# Start crawling and discover pages for training

POST https://services.leadconnectorhq.com/knowledge-bases/crawler

Starts a website crawl that discovers pages for later AI training.
WHEN TO USE: use this when you need to scan a site before training; use trainDiscoveredUrls to ingest selected pages; use getCrawlingStatusForLatestOperation to poll progress.
RETURNS: the discovery operation id, current status, and the URL being crawled.

## Authentication

Send `Authorization: Bearer <token>` using:

- **Sub-Account token: OAuth access token or Private Integration Token.** Scopes: `knowledge-bases.write`

## Parameters

### Header Parameters

| Name | Type | Required | Description |
|------|------|----------|-------------|
| `Version` | string | Yes | API Version |

## OpenAPI Specification

```json
{
  "openapi": "3.0.0",
  "info": {
    "title": "LeadConnector API",
    "version": "v3"
  },
  "servers": [
    {
      "url": "https://services.leadconnectorhq.com"
    }
  ],
  "paths": {
    "/knowledge-bases/crawler": {
      "post": {
        "summary": "Start crawling and discover pages for training",
        "operationId": "startCrawlingAndDiscoverPagesForTraining",
        "description": "Starts a website crawl that discovers pages for later AI training.\nWHEN TO USE: use this when you need to scan a site before training; use trainDiscoveredUrls to ingest selected pages; use getCrawlingStatusForLatestOperation to poll progress.\nRETURNS: the discovery operation id, current status, and the URL being crawled.",
        "parameters": [
          {
            "name": "Version",
            "in": "header",
            "description": "API Version",
            "required": true,
            "schema": {
              "type": "string",
              "enum": [
                "v3"
              ]
            }
          }
        ],
        "requestBody": {
          "required": true,
          "content": {
            "application/json": {
              "schema": {
                "$ref": "#/components/schemas/DiscoverWebsiteRequestDTO"
              }
            }
          }
        },
        "responses": {
          "201": {
            "description": "Crawling and discovery started successfully",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/DiscoverWebsiteResponseDTO"
                }
              }
            }
          },
          "400": {
            "description": "Bad Request",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/BadRequestDTO"
                }
              }
            }
          },
          "401": {
            "description": "Unauthorized",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/UnauthorizedDTO"
                }
              }
            }
          },
          "422": {
            "description": "Unprocessable Entity",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/UnprocessableDTO"
                }
              }
            }
          },
          "500": {
            "description": "Internal Server Error",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/InternalServerErrorDTO"
                }
              }
            }
          }
        },
        "security": [
          {
            "Location-Access": [
              "knowledge-bases.write"
            ]
          }
        ]
      }
    }
  },
  "components": {
    "securitySchemes": {
      "bearer": {
        "scheme": "bearer",
        "bearerFormat": "JWT",
        "name": "Authorization",
        "in": "header",
        "description": "Use the Access Token generated with user type as Sub-Account (OR) Private Integration Token of Sub-Account.",
        "type": "http"
      },
      "Location-Access": {
        "scheme": "bearer",
        "bearerFormat": "JWT",
        "name": "Authorization",
        "in": "header",
        "description": "Use the Access Token generated with user type as Sub-Account (OR) Private Integration Token of Sub-Account.",
        "type": "http"
      },
      "Location-Access-Only": {
        "scheme": "bearer",
        "bearerFormat": "JWT",
        "name": "Authorization",
        "in": "header",
        "description": "Use the Access Token generated with user type as Sub-Account.",
        "type": "http"
      },
      "Agency-Access": {
        "scheme": "bearer",
        "bearerFormat": "JWT",
        "name": "Authorization",
        "in": "header",
        "description": "Use the Access Token generated with user type as Agency (OR) Private Integration Token of Agency.",
        "type": "http"
      },
      "Agency-Access-Only": {
        "scheme": "bearer",
        "bearerFormat": "JWT",
        "name": "Authorization",
        "in": "header",
        "description": "Use the Access Token generated with user type as Agency.",
        "type": "http"
      }
    },
    "schemas": {
      "DiscoverWebsiteRequestDTO": {
        "type": "object",
        "properties": {
          "locationId": {
            "type": "string",
            "description": "Location ID as string",
            "example": "tDtDnQdgm2LXpyiqYvZ6"
          },
          "url": {
            "type": "string",
            "description": "Website URL as string",
            "example": "https://kubernetes.io/tDtDnQdgm2LXpyiqYvZ6"
          },
          "option": {
            "type": "string",
            "description": "Mode as string",
            "example": "Exact",
            "enum": [
              "Exact",
              "Path",
              "Domain"
            ]
          },
          "knowledgeBaseId": {
            "type": "string",
            "description": "knowledge base ID as string",
            "example": "tDtDnQdgm2LXpyiqYvZ6"
          },
          "preSelectedUrls": {
            "description": "Pre-selected URLs from sitemap preview — when provided, only these URLs are crawled",
            "example": [
              "https://example.com/page-1",
              "https://example.com/page-2"
            ],
            "type": "array",
            "items": {
              "type": "string"
            }
          },
          "autoTrain": {
            "type": "boolean",
            "description": "Automatically train the discovered URLs after crawling",
            "example": false
          }
        },
        "required": [
          "locationId",
          "url",
          "option",
          "knowledgeBaseId"
        ],
        "title": "DiscoverWebsiteRequestDTO"
      },
      "DiscoverWebsiteResponseDTO": {
        "type": "object",
        "properties": {
          "operationId": {
            "type": "string",
            "description": "Operation ID for tracking the discovery process",
            "example": "op_abc123xyz"
          },
          "status": {
            "type": "string",
            "description": "Current status of the website discovery operation",
            "example": "Processing",
            "enum": [
              "Pending",
              "Processing",
              "Successful",
              "Failed",
              "Existing",
              "Restricted",
              "Cancelled",
              "Aborted",
              "Training"
            ],
            "readOnly": true
          },
          "url": {
            "type": "string",
            "description": "The URL being discovered/crawled",
            "example": "https://example.com"
          }
        },
        "required": [
          "operationId",
          "status",
          "url"
        ],
        "title": "DiscoverWebsiteResponseDTO"
      },
      "BadRequestDTO": {
        "type": "object",
        "properties": {
          "statusCode": {
            "type": "number",
            "example": 400
          },
          "message": {
            "type": "string",
            "example": "Bad Request"
          }
        },
        "title": "BadRequestDTO"
      },
      "UnauthorizedDTO": {
        "type": "object",
        "properties": {
          "statusCode": {
            "type": "number",
            "example": 401
          },
          "message": {
            "type": "string",
            "example": "Invalid token: access token is invalid"
          },
          "error": {
            "type": "string",
            "example": "Unauthorized"
          }
        },
        "title": "UnauthorizedDTO"
      },
      "UnprocessableDTO": {
        "type": "object",
        "properties": {
          "statusCode": {
            "type": "number",
            "example": 422
          },
          "message": {
            "example": [
              "Unprocessable Entity"
            ],
            "type": "array",
            "items": {
              "type": "string"
            }
          },
          "error": {
            "type": "string",
            "example": "Unprocessable Entity"
          }
        },
        "title": "UnprocessableDTO"
      },
      "InternalServerErrorDTO": {
        "type": "object",
        "properties": {
          "statusCode": {
            "type": "number",
            "description": "HTTP status code",
            "example": 500
          },
          "message": {
            "type": "string",
            "description": "Error message",
            "example": "Internal Server Error"
          }
        },
        "title": "InternalServerErrorDTO"
      }
    }
  }
}
```

## Code Examples

**cURL**

```bash
curl -L -X POST 'https://services.leadconnectorhq.com/knowledge-bases/crawler' \
  -H 'Authorization: Bearer <TOKEN>' \
  -H 'Version: v3' \
  -H 'Content-Type: application/json' \
  --data-raw '{
  "locationId": "tDtDnQdgm2LXpyiqYvZ6",
  "url": "https://kubernetes.io/tDtDnQdgm2LXpyiqYvZ6",
  "option": "Exact",
  "knowledgeBaseId": "tDtDnQdgm2LXpyiqYvZ6",
  "preSelectedUrls": [
    "https://example.com/page-1",
    "https://example.com/page-2"
  ],
  "autoTrain": false
}'
```

**Node.js**

```javascript
const options = {
  method: 'POST',
  headers: {
    Authorization: 'Bearer <TOKEN>',
    Version: 'v3',
    'Content-Type': 'application/json'
  },
  body: JSON.stringify({
      "locationId": "tDtDnQdgm2LXpyiqYvZ6",
      "url": "https://kubernetes.io/tDtDnQdgm2LXpyiqYvZ6",
      "option": "Exact",
      "knowledgeBaseId": "tDtDnQdgm2LXpyiqYvZ6",
      "preSelectedUrls": [
        "https://example.com/page-1",
        "https://example.com/page-2"
      ],
      "autoTrain": false
  })
};

try {
  const response = await fetch('https://services.leadconnectorhq.com/knowledge-bases/crawler', options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}
```

**Python**

```python
import requests

url = "https://services.leadconnectorhq.com/knowledge-bases/crawler"

headers = {
    "Authorization": "Bearer <TOKEN>",
    "Version": "v3"
}

payload = {
    "locationId": "tDtDnQdgm2LXpyiqYvZ6",
    "url": "https://kubernetes.io/tDtDnQdgm2LXpyiqYvZ6",
    "option": "Exact",
    "knowledgeBaseId": "tDtDnQdgm2LXpyiqYvZ6",
    "preSelectedUrls": [
        "https://example.com/page-1",
        "https://example.com/page-2"
    ],
    "autoTrain": false
}

response = requests.post(url, json=payload, headers=headers)
print(response.json())
```

**PHP**

```php
<?php
$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => 'https://services.leadconnectorhq.com/knowledge-bases/crawler',
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_CUSTOMREQUEST => 'POST',
  CURLOPT_HTTPHEADER => [
    'Authorization: Bearer <TOKEN>',
    'Version: v3',
    'Content-Type: application/json',
  ],
  CURLOPT_POSTFIELDS => '{
  "locationId": "tDtDnQdgm2LXpyiqYvZ6",
  "url": "https://kubernetes.io/tDtDnQdgm2LXpyiqYvZ6",
  "option": "Exact",
  "knowledgeBaseId": "tDtDnQdgm2LXpyiqYvZ6",
  "preSelectedUrls": [
    "https://example.com/page-1",
    "https://example.com/page-2"
  ],
  "autoTrain": false
}',
]);

$response = curl_exec($curl);
curl_close($curl);
echo $response;
```

**Java**

```java
HttpResponse<String> response = Unirest
  .post("https://services.leadconnectorhq.com/knowledge-bases/crawler")
  .header("Authorization", "Bearer <TOKEN>")
  .header("Version", "v3")
  .header("Content-Type", "application/json")
  .body("{\"locationId\":\"tDtDnQdgm2LXpyiqYvZ6\",\"url\":\"https://kubernetes.io/tDtDnQdgm2LXpyiqYvZ6\",\"option\":\"Exact\",\"knowledgeBaseId\":\"tDtDnQdgm2LXpyiqYvZ6\",\"preSelectedUrls\":[\"https://example.com/page-1\",\"https://example.com/page-2\"],\"autoTrain\":false}")
  .asString();
```

**Go**

```go
package main

import (
  "fmt"
  "net/http"
  "io"
  "strings"
)

func main() {
  url := "https://services.leadconnectorhq.com/knowledge-bases/crawler"
  payload := strings.NewReader(`{
  "locationId": "tDtDnQdgm2LXpyiqYvZ6",
  "url": "https://kubernetes.io/tDtDnQdgm2LXpyiqYvZ6",
  "option": "Exact",
  "knowledgeBaseId": "tDtDnQdgm2LXpyiqYvZ6",
  "preSelectedUrls": [
    "https://example.com/page-1",
    "https://example.com/page-2"
  ],
  "autoTrain": false
}`)
  req, _ := http.NewRequest("POST", url, payload)
  req.Header.Add("Authorization", "Bearer <TOKEN>")
  req.Header.Add("Version", "v3")

  res, _ := http.DefaultClient.Do(req)
  defer res.Body.Close()
  body, _ := io.ReadAll(res.Body)
  fmt.Println(string(body))
}
```

**Ruby**

```ruby
require 'uri'
require 'net/http'

url = URI("https://services.leadconnectorhq.com/knowledge-bases/crawler")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = "Bearer <TOKEN>"
request["Version"] = "v3"
request["Content-Type"] = "application/json"
request.body = '{
  "locationId": "tDtDnQdgm2LXpyiqYvZ6",
  "url": "https://kubernetes.io/tDtDnQdgm2LXpyiqYvZ6",
  "option": "Exact",
  "knowledgeBaseId": "tDtDnQdgm2LXpyiqYvZ6",
  "preSelectedUrls": [
    "https://example.com/page-1",
    "https://example.com/page-2"
  ],
  "autoTrain": false
}'

response = http.request(request)
puts response.read_body
```

**PowerShell**

```powershell
$headers = @{
  "Authorization" = "Bearer <TOKEN>"
  "Version" = "v3"
}

$body = '{
  "locationId": "tDtDnQdgm2LXpyiqYvZ6",
  "url": "https://kubernetes.io/tDtDnQdgm2LXpyiqYvZ6",
  "option": "Exact",
  "knowledgeBaseId": "tDtDnQdgm2LXpyiqYvZ6",
  "preSelectedUrls": [
    "https://example.com/page-1",
    "https://example.com/page-2"
  ],
  "autoTrain": false
}'

$response = Invoke-RestMethod -Uri 'https://services.leadconnectorhq.com/knowledge-bases/crawler' -Method 'POST' -Headers $headers -Body $body -ContentType 'application/json'
$response | ConvertTo-Json
```