cc91427789
Python crawlers for VIC Register, Funerals Australia, NFDA n8n workflows for scheduled discovery and enrichment SQLite schema and seeded dev database (1,463 providers) End-to-end process documentation in n8n/PROCESS.md
143 lines
4.6 KiB
JSON
143 lines
4.6 KiB
JSON
{
|
|
"name": "1. Weekly Provider Discovery",
|
|
"nodes": [
|
|
{
|
|
"parameters": {
|
|
"rule": {
|
|
"interval": [{ "field": "weeks", "weeksInterval": 1, "triggerAtDay": 1, "triggerAtHour": 2 }]
|
|
}
|
|
},
|
|
"id": "schedule",
|
|
"name": "Weekly Schedule",
|
|
"type": "n8n-nodes-base.scheduleTrigger",
|
|
"typeVersion": 1.2,
|
|
"position": [200, 300]
|
|
},
|
|
{
|
|
"parameters": {
|
|
"command": "cd /opt/crawlers && python3 crawl_vic_register.py 2>&1"
|
|
},
|
|
"id": "crawl_vic",
|
|
"name": "Crawl VIC Register",
|
|
"type": "n8n-nodes-base.executeCommand",
|
|
"typeVersion": 1,
|
|
"position": [450, 140]
|
|
},
|
|
{
|
|
"parameters": {
|
|
"command": "cd /opt/crawlers && python3 crawl_funerals_australia.py 2>&1"
|
|
},
|
|
"id": "crawl_fa",
|
|
"name": "Crawl Funerals Australia",
|
|
"type": "n8n-nodes-base.executeCommand",
|
|
"typeVersion": 1,
|
|
"position": [450, 300]
|
|
},
|
|
{
|
|
"parameters": {
|
|
"command": "cd /opt/crawlers && python3 crawl_nfda.py 2>&1"
|
|
},
|
|
"id": "crawl_nfda",
|
|
"name": "Crawl NFDA",
|
|
"type": "n8n-nodes-base.executeCommand",
|
|
"typeVersion": 1,
|
|
"position": [450, 460]
|
|
},
|
|
{
|
|
"parameters": {
|
|
"mode": "passthrough"
|
|
},
|
|
"id": "merge_crawls",
|
|
"name": "Wait for Crawlers",
|
|
"type": "n8n-nodes-base.merge",
|
|
"typeVersion": 3,
|
|
"position": [700, 300]
|
|
},
|
|
{
|
|
"parameters": {
|
|
"command": "cd /opt/crawlers && python3 dedup.py 2>&1"
|
|
},
|
|
"id": "dedup",
|
|
"name": "Deduplicate & Merge",
|
|
"type": "n8n-nodes-base.executeCommand",
|
|
"typeVersion": 1,
|
|
"position": [950, 300]
|
|
},
|
|
{
|
|
"parameters": {
|
|
"command": "cd /opt/crawlers && python3 -c \"from base import get_db; db=get_db(); r=db.execute('SELECT COUNT(*) as n FROM funeral_brand WHERE listing_tier=\\'listed\\' AND created_at > datetime(\\'now\\', \\'-7 days\\')').fetchone(); print(r['n'])\" 2>&1"
|
|
},
|
|
"id": "count_new",
|
|
"name": "Count New Providers",
|
|
"type": "n8n-nodes-base.executeCommand",
|
|
"typeVersion": 1,
|
|
"position": [1200, 300]
|
|
},
|
|
{
|
|
"parameters": {
|
|
"conditions": {
|
|
"options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict" },
|
|
"conditions": [
|
|
{
|
|
"id": "new_check",
|
|
"leftValue": "={{ $json.stdout.trim() }}",
|
|
"rightValue": "0",
|
|
"operator": { "type": "string", "operation": "notEquals" }
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"id": "has_new",
|
|
"name": "Any New Providers?",
|
|
"type": "n8n-nodes-base.if",
|
|
"typeVersion": 2.2,
|
|
"position": [1450, 300]
|
|
},
|
|
{
|
|
"parameters": {
|
|
"jsCode": "const count = $input.first().json.stdout.trim();\nreturn [{ json: { message: `Weekly discovery complete. ${count} new providers added to the database. They are queued for website discovery and enrichment.`, count: parseInt(count) } }];"
|
|
},
|
|
"id": "summary",
|
|
"name": "Build Summary",
|
|
"type": "n8n-nodes-base.code",
|
|
"typeVersion": 2,
|
|
"position": [1700, 240]
|
|
},
|
|
{
|
|
"parameters": {
|
|
"jsCode": "return [{ json: { message: 'Weekly discovery complete. No new providers found.' } }];"
|
|
},
|
|
"id": "no_new",
|
|
"name": "No New Providers",
|
|
"type": "n8n-nodes-base.code",
|
|
"typeVersion": 2,
|
|
"position": [1700, 420]
|
|
}
|
|
],
|
|
"connections": {
|
|
"Weekly Schedule": {
|
|
"main": [
|
|
[
|
|
{ "node": "Crawl VIC Register", "type": "main", "index": 0 },
|
|
{ "node": "Crawl Funerals Australia", "type": "main", "index": 0 },
|
|
{ "node": "Crawl NFDA", "type": "main", "index": 0 }
|
|
]
|
|
]
|
|
},
|
|
"Crawl VIC Register": { "main": [[ { "node": "Wait for Crawlers", "type": "main", "index": 0 } ]] },
|
|
"Crawl Funerals Australia": { "main": [[ { "node": "Wait for Crawlers", "type": "main", "index": 0 } ]] },
|
|
"Crawl NFDA": { "main": [[ { "node": "Wait for Crawlers", "type": "main", "index": 0 } ]] },
|
|
"Wait for Crawlers": { "main": [[ { "node": "Deduplicate & Merge", "type": "main", "index": 0 } ]] },
|
|
"Deduplicate & Merge": { "main": [[ { "node": "Count New Providers", "type": "main", "index": 0 } ]] },
|
|
"Count New Providers": { "main": [[ { "node": "Any New Providers?", "type": "main", "index": 0 } ]] },
|
|
"Any New Providers?": {
|
|
"main": [
|
|
[{ "node": "Build Summary", "type": "main", "index": 0 }],
|
|
[{ "node": "No New Providers", "type": "main", "index": 0 }]
|
|
]
|
|
}
|
|
},
|
|
"settings": { "executionOrder": "v1" },
|
|
"tags": [{ "name": "funeral-arranger" }]
|
|
}
|