# Get all trained page links by knowledge base

GET https://services.leadconnectorhq.com/knowledge-bases/crawler

Lists discovered and trained website pages for a knowledge base, with pagination and optional URL filter.
WHEN TO USE: use this when you need to inspect crawled pages; use discoverWebsite to find new pages; use deleteTrainedUrlsForKnowledgeBase to remove pages the AI should stop using.
RETURNS: a paginated list of crawled URLs with status, title, and identifiers.

## Authentication

Send `Authorization: Bearer <token>` using:

- **Sub-Account token: OAuth access token or Private Integration Token.** Scopes: `knowledge-bases.readonly`

## Parameters

### Header Parameters

| Name | Type | Required | Description |
|------|------|----------|-------------|
| `Version` | string | Yes | API Version |

### Query Parameters

| Name | Type | Required | Description |
|------|------|----------|-------------|
| `knowledgeBaseId` | string | Yes | knowledge base ID as string |
| `locationId` | string | Yes | location ID as string |
| `page` | number | No | Page number |
| `pageLength` | number | No | Records per page |
| `query` | string | No | query to filter on url links |

## OpenAPI Specification

```json
{
  "openapi": "3.0.0",
  "info": {
    "title": "LeadConnector API",
    "version": "v3"
  },
  "servers": [
    {
      "url": "https://services.leadconnectorhq.com"
    }
  ],
  "paths": {
    "/knowledge-bases/crawler": {
      "get": {
        "summary": "Get all trained page links by knowledge base",
        "operationId": "getAllTrainedPageLinksByKnowledgeBase",
        "description": "Lists discovered and trained website pages for a knowledge base, with pagination and optional URL filter.\nWHEN TO USE: use this when you need to inspect crawled pages; use discoverWebsite to find new pages; use deleteTrainedUrlsForKnowledgeBase to remove pages the AI should stop using.\nRETURNS: a paginated list of crawled URLs with status, title, and identifiers.",
        "parameters": [
          {
            "name": "Version",
            "in": "header",
            "description": "API Version",
            "required": true,
            "schema": {
              "type": "string",
              "enum": [
                "v3"
              ]
            }
          },
          {
            "name": "knowledgeBaseId",
            "required": true,
            "in": "query",
            "description": "knowledge base ID as string",
            "schema": {
              "example": "tDtDnQdgm2LXpyiqYvZ6",
              "type": "string"
            }
          },
          {
            "name": "locationId",
            "required": true,
            "in": "query",
            "description": "location ID as string",
            "schema": {
              "example": "tDtDnQdgm2LXpyiqYvZ6",
              "type": "string"
            }
          },
          {
            "name": "page",
            "required": false,
            "in": "query",
            "description": "Page number",
            "schema": {
              "example": 1,
              "type": "number"
            }
          },
          {
            "name": "pageLength",
            "required": false,
            "in": "query",
            "description": "Records per page",
            "schema": {
              "example": 1,
              "type": "number"
            }
          },
          {
            "name": "query",
            "required": false,
            "in": "query",
            "description": "query to filter on url links",
            "schema": {
              "example": "www.example.com/",
              "type": "string"
            }
          }
        ],
        "responses": {
          "200": {
            "description": "Trained page links retrieved successfully",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/GetAllUrlsByKnowledgeBaseResponseDTO"
                }
              }
            }
          },
          "400": {
            "description": "Bad Request",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/BadRequestDTO"
                }
              }
            }
          },
          "401": {
            "description": "Unauthorized",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/UnauthorizedDTO"
                }
              }
            }
          },
          "422": {
            "description": "Unprocessable Entity",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/UnprocessableDTO"
                }
              }
            }
          },
          "500": {
            "description": "Internal Server Error",
            "content": {
              "application/json": {
                "schema": {
                  "$ref": "#/components/schemas/InternalServerErrorDTO"
                }
              }
            }
          }
        },
        "security": [
          {
            "Location-Access": [
              "knowledge-bases.readonly"
            ]
          }
        ]
      }
    }
  },
  "components": {
    "securitySchemes": {
      "bearer": {
        "scheme": "bearer",
        "bearerFormat": "JWT",
        "name": "Authorization",
        "in": "header",
        "description": "Use the Access Token generated with user type as Sub-Account (OR) Private Integration Token of Sub-Account.",
        "type": "http"
      },
      "Location-Access": {
        "scheme": "bearer",
        "bearerFormat": "JWT",
        "name": "Authorization",
        "in": "header",
        "description": "Use the Access Token generated with user type as Sub-Account (OR) Private Integration Token of Sub-Account.",
        "type": "http"
      },
      "Location-Access-Only": {
        "scheme": "bearer",
        "bearerFormat": "JWT",
        "name": "Authorization",
        "in": "header",
        "description": "Use the Access Token generated with user type as Sub-Account.",
        "type": "http"
      },
      "Agency-Access": {
        "scheme": "bearer",
        "bearerFormat": "JWT",
        "name": "Authorization",
        "in": "header",
        "description": "Use the Access Token generated with user type as Agency (OR) Private Integration Token of Agency.",
        "type": "http"
      },
      "Agency-Access-Only": {
        "scheme": "bearer",
        "bearerFormat": "JWT",
        "name": "Authorization",
        "in": "header",
        "description": "Use the Access Token generated with user type as Agency.",
        "type": "http"
      }
    },
    "schemas": {
      "GetAllUrlsByKnowledgeBaseResponseDTO": {
        "type": "object",
        "properties": {
          "count": {
            "type": "number",
            "description": "Total count of URLs in the knowledge base",
            "example": 64
          },
          "urls": {
            "description": "Array of crawled URLs with their details",
            "example": [
              {
                "id": "688c73a25275c513f5f3a7de",
                "url": "https://developer.mozilla.org/en-US/blog",
                "title": "MDN Blog",
                "status": "Successful",
                "locationId": "qIyivCmsuEOSnyoFYEej",
                "knowledgeBaseId": "Arc9QRauPKkSuMJO8D0m",
                "content": "https://storage.googleapis.com/example.txt",
                "contentEditedByUser": false,
                "updatedAt": "2025-08-01T07:58:29.858Z"
              }
            ],
            "type": "array",
            "items": {
              "type": "object",
              "properties": {
                "id": {
                  "type": "string",
                  "description": "Unique identifier for the URL",
                  "example": "688c73a25275c513f5f3a7de"
                },
                "url": {
                  "type": "string",
                  "description": "The actual URL that was crawled",
                  "example": "https://developer.mozilla.org/en-US/blog"
                },
                "title": {
                  "type": "string",
                  "description": "Title of the webpage",
                  "example": "MDN Blog"
                },
                "status": {
                  "type": "string",
                  "description": "Current processing status of the URL",
                  "example": "Successful",
                  "enum": [
                    "Pending",
                    "Processing",
                    "Successful",
                    "Failed",
                    "Existing",
                    "Restricted",
                    "Cancelled",
                    "Aborted",
                    "Training"
                  ]
                },
                "locationId": {
                  "type": "string",
                  "description": "Location ID associated with this URL",
                  "example": "qIyivCmsuEOSnyoFYEej"
                },
                "knowledgeBaseId": {
                  "type": "string",
                  "description": "Knowledge base ID this URL belongs to",
                  "example": "Arc9QRauPKkSuMJO8D0m"
                },
                "content": {
                  "type": "string",
                  "description": "URL to the stored content file",
                  "example": "https://storage.googleapis.com/crm-conversations-ai-staging/conversationAI/2025/8/1/locations/qIyivCmsuEOSnyoFYEej/688c739c88c89f21ad30f63e-4IseusbZ3vDXMT8WNw9M.txt"
                },
                "contentEditedByUser": {
                  "type": "boolean",
                  "description": "Whether the content was edited by user",
                  "example": false
                },
                "updatedAt": {
                  "type": "string",
                  "description": "Last updated timestamp",
                  "example": "2025-08-01T07:58:29.858Z"
                }
              },
              "required": [
                "id",
                "url",
                "title",
                "status",
                "locationId",
                "knowledgeBaseId",
                "content",
                "contentEditedByUser",
                "updatedAt"
              ],
              "title": "CrawledUrlDTO"
            }
          }
        },
        "required": [
          "count",
          "urls"
        ],
        "title": "GetAllUrlsByKnowledgeBaseResponseDTO"
      },
      "BadRequestDTO": {
        "type": "object",
        "properties": {
          "statusCode": {
            "type": "number",
            "example": 400
          },
          "message": {
            "type": "string",
            "example": "Bad Request"
          }
        },
        "title": "BadRequestDTO"
      },
      "UnauthorizedDTO": {
        "type": "object",
        "properties": {
          "statusCode": {
            "type": "number",
            "example": 401
          },
          "message": {
            "type": "string",
            "example": "Invalid token: access token is invalid"
          },
          "error": {
            "type": "string",
            "example": "Unauthorized"
          }
        },
        "title": "UnauthorizedDTO"
      },
      "UnprocessableDTO": {
        "type": "object",
        "properties": {
          "statusCode": {
            "type": "number",
            "example": 422
          },
          "message": {
            "example": [
              "Unprocessable Entity"
            ],
            "type": "array",
            "items": {
              "type": "string"
            }
          },
          "error": {
            "type": "string",
            "example": "Unprocessable Entity"
          }
        },
        "title": "UnprocessableDTO"
      },
      "InternalServerErrorDTO": {
        "type": "object",
        "properties": {
          "statusCode": {
            "type": "number",
            "description": "HTTP status code",
            "example": 500
          },
          "message": {
            "type": "string",
            "description": "Error message",
            "example": "Internal Server Error"
          }
        },
        "title": "InternalServerErrorDTO"
      }
    }
  }
}
```

## Code Examples

**cURL**

```bash
curl -L -X GET 'https://services.leadconnectorhq.com/knowledge-bases/crawler?knowledgeBaseId=<KNOWLEDGEBASEID>&locationId=<LOCATIONID>' \
  -H 'Authorization: Bearer <TOKEN>' \
  -H 'Version: v3'
```

**Node.js**

```javascript
const options = {
  method: 'GET',
  headers: {
    Authorization: 'Bearer <TOKEN>',
    Version: 'v3'
  }
};

try {
  const response = await fetch('https://services.leadconnectorhq.com/knowledge-bases/crawler?knowledgeBaseId=<KNOWLEDGEBASEID>&locationId=<LOCATIONID>', options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}
```

**Python**

```python
import requests

url = "https://services.leadconnectorhq.com/knowledge-bases/crawler"

headers = {
    "Authorization": "Bearer <TOKEN>",
    "Version": "v3"
}

params = {
    "knowledgeBaseId": "<KNOWLEDGEBASEID>",
    "locationId": "<LOCATIONID>"
}

response = requests.get(url, params=params, headers=headers)
print(response.json())
```

**PHP**

```php
<?php
$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => 'https://services.leadconnectorhq.com/knowledge-bases/crawler?knowledgeBaseId=<KNOWLEDGEBASEID>&locationId=<LOCATIONID>',
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_CUSTOMREQUEST => 'GET',
  CURLOPT_HTTPHEADER => [
    'Authorization: Bearer <TOKEN>',
    'Version: v3',
  ],
]);

$response = curl_exec($curl);
curl_close($curl);
echo $response;
```

**Java**

```java
HttpResponse<String> response = Unirest
  .get("https://services.leadconnectorhq.com/knowledge-bases/crawler?knowledgeBaseId=<KNOWLEDGEBASEID>&locationId=<LOCATIONID>")
  .header("Authorization", "Bearer <TOKEN>")
  .header("Version", "v3")
  .asString();
```

**Go**

```go
package main

import (
  "fmt"
  "net/http"
  "io"
)

func main() {
  url := "https://services.leadconnectorhq.com/knowledge-bases/crawler?knowledgeBaseId=<KNOWLEDGEBASEID>&locationId=<LOCATIONID>"
  req, _ := http.NewRequest("GET", url, nil)
  req.Header.Add("Authorization", "Bearer <TOKEN>")
  req.Header.Add("Version", "v3")

  res, _ := http.DefaultClient.Do(req)
  defer res.Body.Close()
  body, _ := io.ReadAll(res.Body)
  fmt.Println(string(body))
}
```

**Ruby**

```ruby
require 'uri'
require 'net/http'

url = URI("https://services.leadconnectorhq.com/knowledge-bases/crawler?knowledgeBaseId=<KNOWLEDGEBASEID>&locationId=<LOCATIONID>")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = "Bearer <TOKEN>"
request["Version"] = "v3"

response = http.request(request)
puts response.read_body
```

**PowerShell**

```powershell
$headers = @{
  "Authorization" = "Bearer <TOKEN>"
  "Version" = "v3"
}

$response = Invoke-RestMethod -Uri 'https://services.leadconnectorhq.com/knowledge-bases/crawler?knowledgeBaseId=<KNOWLEDGEBASEID>&locationId=<LOCATIONID>' -Method 'GET' -Headers $headers
$response | ConvertTo-Json
```