Initial commit: funeral provider discovery pipeline

Python crawlers for VIC Register, Funerals Australia, NFDA n8n workflows for scheduled discovery and enrichment SQLite schema and seeded dev database (1,463 providers) End-to-end process documentation in n8n/PROCESS.md
2026-04-24 10:27:08 +10:00
commit cc91427789
30 changed files with 4706 additions and 0 deletions
--- a/n8n/workflows/1_weekly_discovery.json
+++ b/n8n/workflows/1_weekly_discovery.json
@@ -0,0 +1,142 @@
+{
+  "name": "1. Weekly Provider Discovery",
+  "nodes": [
+    {
+      "parameters": {
+        "rule": {
+          "interval": [{ "field": "weeks", "weeksInterval": 1, "triggerAtDay": 1, "triggerAtHour": 2 }]
+        }
+      },
+      "id": "schedule",
+      "name": "Weekly Schedule",
+      "type": "n8n-nodes-base.scheduleTrigger",
+      "typeVersion": 1.2,
+      "position": [200, 300]
+    },
+    {
+      "parameters": {
+        "command": "cd /opt/crawlers && python3 crawl_vic_register.py 2>&1"
+      },
+      "id": "crawl_vic",
+      "name": "Crawl VIC Register",
+      "type": "n8n-nodes-base.executeCommand",
+      "typeVersion": 1,
+      "position": [450, 140]
+    },
+    {
+      "parameters": {
+        "command": "cd /opt/crawlers && python3 crawl_funerals_australia.py 2>&1"
+      },
+      "id": "crawl_fa",
+      "name": "Crawl Funerals Australia",
+      "type": "n8n-nodes-base.executeCommand",
+      "typeVersion": 1,
+      "position": [450, 300]
+    },
+    {
+      "parameters": {
+        "command": "cd /opt/crawlers && python3 crawl_nfda.py 2>&1"
+      },
+      "id": "crawl_nfda",
+      "name": "Crawl NFDA",
+      "type": "n8n-nodes-base.executeCommand",
+      "typeVersion": 1,
+      "position": [450, 460]
+    },
+    {
+      "parameters": {
+        "mode": "passthrough"
+      },
+      "id": "merge_crawls",
+      "name": "Wait for Crawlers",
+      "type": "n8n-nodes-base.merge",
+      "typeVersion": 3,
+      "position": [700, 300]
+    },
+    {
+      "parameters": {
+        "command": "cd /opt/crawlers && python3 dedup.py 2>&1"
+      },
+      "id": "dedup",
+      "name": "Deduplicate & Merge",
+      "type": "n8n-nodes-base.executeCommand",
+      "typeVersion": 1,
+      "position": [950, 300]
+    },
+    {
+      "parameters": {
+        "command": "cd /opt/crawlers && python3 -c \"from base import get_db; db=get_db(); r=db.execute('SELECT COUNT(*) as n FROM funeral_brand WHERE listing_tier=\\'listed\\' AND created_at > datetime(\\'now\\', \\'-7 days\\')').fetchone(); print(r['n'])\" 2>&1"
+      },
+      "id": "count_new",
+      "name": "Count New Providers",
+      "type": "n8n-nodes-base.executeCommand",
+      "typeVersion": 1,
+      "position": [1200, 300]
+    },
+    {
+      "parameters": {
+        "conditions": {
+          "options": { "caseSensitive": true, "leftValue": "", "typeValidation": "strict" },
+          "conditions": [
+            {
+              "id": "new_check",
+              "leftValue": "={{ $json.stdout.trim() }}",
+              "rightValue": "0",
+              "operator": { "type": "string", "operation": "notEquals" }
+            }
+          ]
+        }
+      },
+      "id": "has_new",
+      "name": "Any New Providers?",
+      "type": "n8n-nodes-base.if",
+      "typeVersion": 2.2,
+      "position": [1450, 300]
+    },
+    {
+      "parameters": {
+        "jsCode": "const count = $input.first().json.stdout.trim();\nreturn [{ json: { message: `Weekly discovery complete. ${count} new providers added to the database. They are queued for website discovery and enrichment.`, count: parseInt(count) } }];"
+      },
+      "id": "summary",
+      "name": "Build Summary",
+      "type": "n8n-nodes-base.code",
+      "typeVersion": 2,
+      "position": [1700, 240]
+    },
+    {
+      "parameters": {
+        "jsCode": "return [{ json: { message: 'Weekly discovery complete. No new providers found.' } }];"
+      },
+      "id": "no_new",
+      "name": "No New Providers",
+      "type": "n8n-nodes-base.code",
+      "typeVersion": 2,
+      "position": [1700, 420]
+    }
+  ],
+  "connections": {
+    "Weekly Schedule": {
+      "main": [
+        [
+          { "node": "Crawl VIC Register", "type": "main", "index": 0 },
+          { "node": "Crawl Funerals Australia", "type": "main", "index": 0 },
+          { "node": "Crawl NFDA", "type": "main", "index": 0 }
+        ]
+      ]
+    },
+    "Crawl VIC Register": { "main": [[ { "node": "Wait for Crawlers", "type": "main", "index": 0 } ]] },
+    "Crawl Funerals Australia": { "main": [[ { "node": "Wait for Crawlers", "type": "main", "index": 0 } ]] },
+    "Crawl NFDA": { "main": [[ { "node": "Wait for Crawlers", "type": "main", "index": 0 } ]] },
+    "Wait for Crawlers": { "main": [[ { "node": "Deduplicate & Merge", "type": "main", "index": 0 } ]] },
+    "Deduplicate & Merge": { "main": [[ { "node": "Count New Providers", "type": "main", "index": 0 } ]] },
+    "Count New Providers": { "main": [[ { "node": "Any New Providers?", "type": "main", "index": 0 } ]] },
+    "Any New Providers?": {
+      "main": [
+        [{ "node": "Build Summary", "type": "main", "index": 0 }],
+        [{ "node": "No New Providers", "type": "main", "index": 0 }]
+      ]
+    }
+  },
+  "settings": { "executionOrder": "v1" },
+  "tags": [{ "name": "funeral-arranger" }]
+}