{
  "name": "Docs site to Dify knowledge base",
  "nodes": [
    {
      "parameters": {},
      "type": "n8n-nodes-base.manualTrigger",
      "typeVersion": 1,
      "position": [
        0,
        0
      ],
      "name": "Run manually"
    },
    {
      "parameters": {
        "method": "GET",
        "url": "https://YOUR-DIFY-HOST/v1/datasets/YOUR_DATASET_ID/documents?limit=100",
        "authentication": "genericCredentialType",
        "genericAuthType": "httpBearerAuth",
        "options": {}
      },
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "position": [
        220,
        0
      ],
      "name": "List existing documents",
      "credentials": {
        "httpBearerAuth": {
          "id": "",
          "name": "Dify Knowledge API key"
        }
      }
    },
    {
      "parameters": {
        "conditions": {
          "options": {
            "caseSensitive": true,
            "leftValue": "",
            "typeValidation": "strict",
            "version": 2
          },
          "conditions": [
            {
              "id": "i1",
              "leftValue": "={{ $json.total }}",
              "rightValue": 0,
              "operator": {
                "type": "number",
                "operation": "gt"
              }
            }
          ],
          "combinator": "and"
        },
        "options": {}
      },
      "type": "n8n-nodes-base.if",
      "typeVersion": 2.2,
      "position": [
        440,
        0
      ],
      "name": "Any old documents?"
    },
    {
      "parameters": {
        "fieldToSplitOut": "data",
        "options": {}
      },
      "type": "n8n-nodes-base.splitOut",
      "typeVersion": 1,
      "position": [
        660,
        -120
      ],
      "name": "One item per old document"
    },
    {
      "parameters": {
        "method": "DELETE",
        "url": "=https://YOUR-DIFY-HOST/v1/datasets/YOUR_DATASET_ID/documents/{{ $json.id }}",
        "authentication": "genericCredentialType",
        "genericAuthType": "httpBearerAuth",
        "options": {}
      },
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "position": [
        880,
        -120
      ],
      "name": "Delete old document",
      "credentials": {
        "httpBearerAuth": {
          "id": "",
          "name": "Dify Knowledge API key"
        }
      }
    },
    {
      "parameters": {
        "operation": "crawl",
        "crawl_url": "https://webcrawlerapi.com/docs",
        "items_limit": 20,
        "whitelist_regexp": "^https://webcrawlerapi\\.com/docs/.*",
        "blacklist_regexp": "",
        "respect_robots_txt": true,
        "max_depth": 2,
        "output_as_file": false
      },
      "type": "n8n-nodes-webcrawlerapi.webCrawlerApiNode",
      "typeVersion": 1,
      "position": [
        1100,
        0
      ],
      "name": "Crawl docs site",
      "credentials": {
        "webCrawlerApi": {
          "id": "",
          "name": "WebCrawlerAPI account"
        }
      },
      "executeOnce": true
    },
    {
      "parameters": {
        "jsCode": "// The WebCrawlerAPI node returns ONE item: the job plus one combined markdown string.\n// Pages are separated by a small header: ----\\nurl: <page url>\\n----\nconst { markdown, job } = $input.first().json;\nconst titles = Object.fromEntries(job.job_items.map((i) => [i.original_url, i.title]));\n\nconst parts = markdown.split(/^----\\nurl: (\\S+)\\n----\\n/m);\nconst pages = [];\nfor (let i = 1; i < parts.length; i += 2) {\n  pages.push({\n    json: {\n      url: parts[i],\n      title: titles[parts[i]] || '',\n      markdown: parts[i + 1].trim(),\n      last_crawled_at: job.finished_at,\n    },\n  });\n}\nreturn pages;"
      },
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "position": [
        1320,
        0
      ],
      "name": "One item per page"
    },
    {
      "parameters": {
        "conditions": {
          "options": {
            "caseSensitive": true,
            "leftValue": "",
            "typeValidation": "strict",
            "version": 2
          },
          "conditions": [
            {
              "id": "c1",
              "leftValue": "={{ $json.markdown.length }}",
              "rightValue": 300,
              "operator": {
                "type": "number",
                "operation": "gt"
              }
            }
          ],
          "combinator": "and"
        },
        "options": {}
      },
      "type": "n8n-nodes-base.filter",
      "typeVersion": 2.2,
      "position": [
        1540,
        0
      ],
      "name": "Drop near-empty pages"
    },
    {
      "parameters": {
        "method": "POST",
        "url": "https://YOUR-DIFY-HOST/v1/datasets/YOUR_DATASET_ID/document/create-by-text",
        "authentication": "genericCredentialType",
        "genericAuthType": "httpBearerAuth",
        "options": {
          "batching": {
            "batch": {
              "batchSize": 1,
              "batchInterval": 300
            }
          }
        },
        "sendBody": true,
        "specifyBody": "json",
        "jsonBody": "={{ JSON.stringify({ name: $json.url, text: $json.markdown, indexing_technique: 'economy', process_rule: { mode: 'automatic' } }) }}"
      },
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "position": [
        1760,
        0
      ],
      "name": "Create Dify document",
      "credentials": {
        "httpBearerAuth": {
          "id": "",
          "name": "Dify Knowledge API key"
        }
      }
    }
  ],
  "connections": {
    "Run manually": {
      "main": [
        [
          {
            "node": "List existing documents",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "List existing documents": {
      "main": [
        [
          {
            "node": "Any old documents?",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Any old documents?": {
      "main": [
        [
          {
            "node": "One item per old document",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "Crawl docs site",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "One item per old document": {
      "main": [
        [
          {
            "node": "Delete old document",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Delete old document": {
      "main": [
        [
          {
            "node": "Crawl docs site",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Crawl docs site": {
      "main": [
        [
          {
            "node": "One item per page",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "One item per page": {
      "main": [
        [
          {
            "node": "Drop near-empty pages",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Drop near-empty pages": {
      "main": [
        [
          {
            "node": "Create Dify document",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  },
  "settings": {
    "executionOrder": "v1"
  },
  "active": false,
  "pinData": {}
}