# Get crawling status for the latest operation

GET https://services.leadconnectorhq.com/knowledge-bases/crawler/status

Fetches crawl/train progress for a specific operation, or the latest operation when operationId is omitted.
WHEN TO USE: use this when you need to poll a discoverWebsite or trainDiscoveredUrls run; do not use it to start a crawl or train pages.
RETURNS: aggregated counts by status plus detailed operation information.

## Authentication

Send `Authorization: Bearer <token>` using:

- **Sub-Account token: OAuth access token or Private Integration Token.** Scopes: `knowledge-bases.readonly`

## Parameters

### Header Parameters

| Name | Type | Required | Description |
|------|------|----------|-------------|
| `Version` | string | Yes | API Version |

### Query Parameters

| Name | Type | Required | Description |
|------|------|----------|-------------|
| `locationId` | string | Yes | Location ID as string |
| `operationId` | string | No | operation id as string (optional - gets latest if not provided) |
| `knowledgeBaseId` | string | Yes | knowledge base id |

## OpenAPI Specification

```json
{
  "openapi": "3.0.0",
  "info": {
    "title": "LeadConnector API",
    "version": "v3"
  },
  "servers": [
    {
      "url": "https://services.leadconnectorhq.com"
    }
  ],
  "paths": {
    "/knowledge-bases/crawler/status": {
      "get": {
        "summary": "Get crawling status for the latest operation",
        "operationId": "getCrawlingStatusForTheLatestOperation",
        "description": "Fetches crawl/train progress for a specific operation, or the latest operation when operationId is omitted.\nWHEN TO USE: use this when you need to poll a discoverWebsite or trainDiscoveredUrls run; do not use it to start a crawl or train pages.\nRETURNS: aggregated counts by status plus detailed operation information.",
        "parameters": [
          {
            "name": "Version",
            "in": "header",
            "description": "API Version",
            "required": true,
            "schema": {
              "type": "string",
              "enum": [
                "v3"
              ]
            }
          },
          {
            "name": "locationId",
            "required": true,
            "in": "query",
            "description": "Location ID as string",
            "schema": {
              "example": "tDtDnQdgm2LXpyiqYvZ6",
              "type": "string"
            }
          },
          {
            "name": "operationId",
            "required": false,
            "in": "query",
            "description": "operation id as string (optional - gets latest if not provided)",
            "schema": {
              "example": "688b640bcb02d498102a13f0",
              "type": "string"
            }
          },
          {
            "name": "knowledgeBaseId",
            "required": true,
            "in": "query",
            "description": "knowledge base id",
            "schema": {
              "example": "RXzAdYGknD2phAoln3Jl",
              "type": "string"
            }
          }
        ],
        "responses": {
          "200": {
            "description": "Operation status fetched successfully",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/CrawlingStatusResponseDTO"
                }
              }
            }
          },
          "400": {
            "description": "Bad Request",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/BadRequestDTO"
                }
              }
            }
          },
          "401": {
            "description": "Unauthorized",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/UnauthorizedDTO"
                }
              }
            }
          },
          "422": {
            "description": "Unprocessable Entity",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/UnprocessableDTO"
                }
              }
            }
          },
          "500": {
            "description": "Internal Server Error",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/InternalServerErrorDTO"
                }
              }
            }
          }
        },
        "security": [
          {
            "Location-Access": [
              "knowledge-bases.readonly"
            ]
          }
        ]
      }
    }
  },
  "components": {
    "securitySchemes": {
      "bearer": {
        "scheme": "bearer",
        "bearerFormat": "JWT",
        "name": "Authorization",
        "in": "header",
        "description": "Use the Access Token generated with user type as Sub-Account (OR) Private Integration Token of Sub-Account.",
        "type": "http"
      },
      "Location-Access": {
        "scheme": "bearer",
        "bearerFormat": "JWT",
        "name": "Authorization",
        "in": "header",
        "description": "Use the Access Token generated with user type as Sub-Account (OR) Private Integration Token of Sub-Account.",
        "type": "http"
      },
      "Location-Access-Only": {
        "scheme": "bearer",
        "bearerFormat": "JWT",
        "name": "Authorization",
        "in": "header",
        "description": "Use the Access Token generated with user type as Sub-Account.",
        "type": "http"
      },
      "Agency-Access": {
        "scheme": "bearer",
        "bearerFormat": "JWT",
        "name": "Authorization",
        "in": "header",
        "description": "Use the Access Token generated with user type as Agency (OR) Private Integration Token of Agency.",
        "type": "http"
      },
      "Agency-Access-Only": {
        "scheme": "bearer",
        "bearerFormat": "JWT",
        "name": "Authorization",
        "in": "header",
        "description": "Use the Access Token generated with user type as Agency.",
        "type": "http"
      }
    },
    "schemas": {
      "CrawlingStatusResponseDTO": {
        "type": "object",
        "properties": {
          "aggregate": {
            "description": "Aggregated crawling results by status",
            "example": [
              {
                "_id": "Failed",
                "records": [
                  {
                    "url": "https://developer.mozilla.org/en-US/blog/rss.xml",
                    "id": "688e41118a188704914d13c0"
                  }
                ]
              }
            ],
            "type": "array",
            "items": {
              "type": "object",
              "properties": {
                "records": {
                  "description": "Array of records for this status",
                  "example": [
                    {
                      "url": "https://developer.mozilla.org/en-US/blog/rss.xml",
                      "id": "688e41118a188704914d13c0"
                    }
                  ],
                  "type": "array",
                  "items": {
                    "type": "object",
                    "properties": {
                      "url": {
                        "type": "string",
                        "description": "URL being crawled",
                        "example": "https://developer.mozilla.org/en-US/blog/rss.xml"
                      },
                      "id": {
                        "type": "string",
                        "description": "Unique record identifier",
                        "example": "688e41118a188704914d13c0"
                      },
                      "title": {
                        "type": "string",
                        "description": "Page title (for successful/pending records)",
                        "example": "JavaScript Temporal is coming | MDN Blog"
                      },
                      "error": {
                        "type": "object",
                        "description": "Error details (supports both structured and legacy formats)",
                        "example": {}
                      }
                    },
                    "required": [
                      "url",
                      "id"
                    ],
                    "title": "CrawlingRecordDTO"
                  }
                }
              },
              "required": [
                "records"
              ],
              "title": "CrawlingAggregateDTO"
            }
          },
          "operationDetails": {
            "description": "Detailed operation information",
            "example": {},
            "allOf": [
              {
                "type": "object",
                "properties": {
                  "discoveredUrlsCount": {
                    "type": "number",
                    "description": "Number of URLs discovered",
                    "example": 66
                  },
                  "selectedUrlsCount": {
                    "type": "number",
                    "description": "Number of URLs selected for sitemap training (0 or absent = discovery/link-following mode)",
                    "example": 4
                  },
                  "trainedUrlsCount": {
                    "type": "number",
                    "description": "Number of URLs successfully trained",
                    "example": 0
                  },
                  "locationId": {
                    "type": "string",
                    "description": "Associated location ID",
                    "example": "nnAzVJqSv6PJ1p6zrhvC"
                  },
                  "status": {
                    "type": "string",
                    "description": "Current operation status",
                    "example": "Pending",
                    "enum": [
                      "Pending",
                      "Processing",
                      "Successful",
                      "Failed",
                      "Existing",
                      "Restricted",
                      "Cancelled",
                      "Aborted",
                      "Training"
                    ]
                  },
                  "url": {
                    "type": "string",
                    "description": "Base URL being crawled",
                    "example": "https://developer.mozilla.org/en-US/blog/"
                  },
                  "mode": {
                    "type": "string",
                    "description": "Crawling mode used",
                    "example": "Path",
                    "enum": [
                      "Exact",
                      "Path",
                      "Domain"
                    ]
                  },
                  "knowledgeBaseId": {
                    "type": "string",
                    "description": "Knowledge base ID",
                    "example": "CCNPhKqSCkTOG8O7dtbq"
                  },
                  "createdAt": {
                    "type": "string",
                    "description": "Operation creation timestamp",
                    "example": "2025-08-02T16:47:08.182Z"
                  },
                  "updatedAt": {
                    "type": "string",
                    "description": "Last update timestamp",
                    "example": "2025-08-02T16:48:43.635Z"
                  },
                  "robotsFileData": {
                    "type": "string",
                    "description": "Robots.txt file content",
                    "example": "User-agent: *\nSitemap: https://developer.mozilla.org/sitemap.xml\nDisallow: /api/"
                  },
                  "source": {
                    "type": "string",
                    "description": "Source of the operation (manual, automated, or snapshot)",
                    "example": "manual",
                    "enum": [
                      "manual",
                      "automated",
                      "snapshot"
                    ]
                  },
                  "parentOperationId": {
                    "type": "string",
                    "description": "Parent operation ID (for automated operations that refresh URLs from Domain/Path crawls)",
                    "example": "688e410c8a18870ecf4d13bb"
                  },
                  "totalSyncAttempts": {
                    "type": "number",
                    "description": "Total number of sync attempts for automated operations",
                    "example": 5
                  },
                  "lastSyncStartedAt": {
                    "type": "string",
                    "description": "Timestamp of last sync start for automated operations",
                    "example": "2026-01-28T10:00:00Z"
                  }
                },
                "required": [
                  "discoveredUrlsCount",
                  "trainedUrlsCount",
                  "locationId",
                  "status",
                  "url",
                  "mode",
                  "knowledgeBaseId",
                  "createdAt",
                  "updatedAt"
                ],
                "title": "OperationDetailsDTO"
              }
            ]
          }
        },
        "required": [
          "aggregate",
          "operationDetails"
        ],
        "title": "CrawlingStatusResponseDTO"
      },
      "BadRequestDTO": {
        "type": "object",
        "properties": {
          "statusCode": {
            "type": "number",
            "example": 400
          },
          "message": {
            "type": "string",
            "example": "Bad Request"
          }
        },
        "title": "BadRequestDTO"
      },
      "UnauthorizedDTO": {
        "type": "object",
        "properties": {
          "statusCode": {
            "type": "number",
            "example": 401
          },
          "message": {
            "type": "string",
            "example": "Invalid token: access token is invalid"
          },
          "error": {
            "type": "string",
            "example": "Unauthorized"
          }
        },
        "title": "UnauthorizedDTO"
      },
      "UnprocessableDTO": {
        "type": "object",
        "properties": {
          "statusCode": {
            "type": "number",
            "example": 422
          },
          "message": {
            "example": [
              "Unprocessable Entity"
            ],
            "type": "array",
            "items": {
              "type": "string"
            }
          },
          "error": {
            "type": "string",
            "example": "Unprocessable Entity"
          }
        },
        "title": "UnprocessableDTO"
      },
      "InternalServerErrorDTO": {
        "type": "object",
        "properties": {
          "statusCode": {
            "type": "number",
            "description": "HTTP status code",
            "example": 500
          },
          "message": {
            "type": "string",
            "description": "Error message",
            "example": "Internal Server Error"
          }
        },
        "title": "InternalServerErrorDTO"
      }
    }
  }
}
```

## Code Examples

**cURL**

```bash
curl -L -X GET 'https://services.leadconnectorhq.com/knowledge-bases/crawler/status?locationId=<LOCATIONID>&knowledgeBaseId=<KNOWLEDGEBASEID>' \
  -H 'Authorization: Bearer <TOKEN>' \
  -H 'Version: v3'
```

**Node.js**

```javascript
const options = {
  method: 'GET',
  headers: {
    Authorization: 'Bearer <TOKEN>',
    Version: 'v3'
  }
};

try {
  const response = await fetch('https://services.leadconnectorhq.com/knowledge-bases/crawler/status?locationId=<LOCATIONID>&knowledgeBaseId=<KNOWLEDGEBASEID>', options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}
```

**Python**

```python
import requests

url = "https://services.leadconnectorhq.com/knowledge-bases/crawler/status"

headers = {
    "Authorization": "Bearer <TOKEN>",
    "Version": "v3"
}

params = {
    "locationId": "<LOCATIONID>",
    "knowledgeBaseId": "<KNOWLEDGEBASEID>"
}

response = requests.get(url, params=params, headers=headers)
print(response.json())
```

**PHP**

```php
<?php
$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => 'https://services.leadconnectorhq.com/knowledge-bases/crawler/status?locationId=<LOCATIONID>&knowledgeBaseId=<KNOWLEDGEBASEID>',
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_CUSTOMREQUEST => 'GET',
  CURLOPT_HTTPHEADER => [
    'Authorization: Bearer <TOKEN>',
    'Version: v3',
  ],
]);

$response = curl_exec($curl);
curl_close($curl);
echo $response;
```

**Java**

```java
HttpResponse<String> response = Unirest
  .get("https://services.leadconnectorhq.com/knowledge-bases/crawler/status?locationId=<LOCATIONID>&knowledgeBaseId=<KNOWLEDGEBASEID>")
  .header("Authorization", "Bearer <TOKEN>")
  .header("Version", "v3")
  .asString();
```

**Go**

```go
package main

import (
  "fmt"
  "net/http"
  "io"
)

func main() {
  url := "https://services.leadconnectorhq.com/knowledge-bases/crawler/status?locationId=<LOCATIONID>&knowledgeBaseId=<KNOWLEDGEBASEID>"
  req, _ := http.NewRequest("GET", url, nil)
  req.Header.Add("Authorization", "Bearer <TOKEN>")
  req.Header.Add("Version", "v3")

  res, _ := http.DefaultClient.Do(req)
  defer res.Body.Close()
  body, _ := io.ReadAll(res.Body)
  fmt.Println(string(body))
}
```

**Ruby**

```ruby
require 'uri'
require 'net/http'

url = URI("https://services.leadconnectorhq.com/knowledge-bases/crawler/status?locationId=<LOCATIONID>&knowledgeBaseId=<KNOWLEDGEBASEID>")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = "Bearer <TOKEN>"
request["Version"] = "v3"

response = http.request(request)
puts response.read_body
```

**PowerShell**

```powershell
$headers = @{
  "Authorization" = "Bearer <TOKEN>"
  "Version" = "v3"
}

$response = Invoke-RestMethod -Uri 'https://services.leadconnectorhq.com/knowledge-bases/crawler/status?locationId=<LOCATIONID>&knowledgeBaseId=<KNOWLEDGEBASEID>' -Method 'GET' -Headers $headers
$response | ConvertTo-Json
```