Initial commit: funeral provider discovery pipeline
Python crawlers for VIC Register, Funerals Australia, NFDA n8n workflows for scheduled discovery and enrichment SQLite schema and seeded dev database (1,463 providers) End-to-end process documentation in n8n/PROCESS.md
This commit is contained in:
65
n8n/workflows/4_monthly_refresh.json
Normal file
65
n8n/workflows/4_monthly_refresh.json
Normal file
@@ -0,0 +1,65 @@
|
||||
{
|
||||
"name": "4. Monthly Re-enrichment",
|
||||
"nodes": [
|
||||
{
|
||||
"parameters": {
|
||||
"rule": {
|
||||
"interval": [{ "field": "months", "monthsInterval": 1, "triggerAtDayOfMonth": 1, "triggerAtHour": 3 }]
|
||||
}
|
||||
},
|
||||
"id": "schedule",
|
||||
"name": "Monthly Schedule",
|
||||
"type": "n8n-nodes-base.scheduleTrigger",
|
||||
"typeVersion": 1.2,
|
||||
"position": [200, 300]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"command": "cd /opt/crawlers && python3 -c \"\nimport sqlite3\ndb = sqlite3.connect('/opt/database/providers.db')\n# Reset enrichment for providers last checked > 30 days ago\nupdated = db.execute('''\n UPDATE funeral_brand\n SET enrichment_status = 'pending',\n updated_at = datetime('now')\n WHERE verified = 0\n AND website IS NOT NULL\n AND last_enriched_at < datetime('now', '-30 days')\n''').rowcount\ndb.commit()\nprint(f'{updated} providers queued for re-enrichment')\n\" 2>&1"
|
||||
},
|
||||
"id": "reset_stale",
|
||||
"name": "Queue Stale Providers",
|
||||
"type": "n8n-nodes-base.executeCommand",
|
||||
"typeVersion": 1,
|
||||
"position": [450, 300]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"command": "cd /opt/crawlers && python3 enrich_websites.py --limit=200 2>&1"
|
||||
},
|
||||
"id": "re_enrich",
|
||||
"name": "Re-enrich (batch 200)",
|
||||
"type": "n8n-nodes-base.executeCommand",
|
||||
"typeVersion": 1,
|
||||
"position": [700, 300]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"command": "cd /opt/crawlers && python3 compute_tiers.py 2>&1"
|
||||
},
|
||||
"id": "recompute",
|
||||
"name": "Recompute Tiers",
|
||||
"type": "n8n-nodes-base.executeCommand",
|
||||
"typeVersion": 1,
|
||||
"position": [950, 300]
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"jsCode": "const output = $input.first().json.stdout || '';\nreturn [{ json: { message: 'Monthly re-enrichment complete.', output } }];"
|
||||
},
|
||||
"id": "summary",
|
||||
"name": "Summary",
|
||||
"type": "n8n-nodes-base.code",
|
||||
"typeVersion": 2,
|
||||
"position": [1200, 300]
|
||||
}
|
||||
],
|
||||
"connections": {
|
||||
"Monthly Schedule": { "main": [[ { "node": "Queue Stale Providers", "type": "main", "index": 0 } ]] },
|
||||
"Queue Stale Providers": { "main": [[ { "node": "Re-enrich (batch 200)", "type": "main", "index": 0 } ]] },
|
||||
"Re-enrich (batch 200)": { "main": [[ { "node": "Recompute Tiers", "type": "main", "index": 0 } ]] },
|
||||
"Recompute Tiers": { "main": [[ { "node": "Summary", "type": "main", "index": 0 } ]] }
|
||||
},
|
||||
"settings": { "executionOrder": "v1" },
|
||||
"tags": [{ "name": "funeral-arranger" }]
|
||||
}
|
||||
Reference in New Issue
Block a user