From 06f78f2d03c7e68ed1cdc0ebf3c7e958d5812cea Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Sun, 14 Jun 2026 17:42:35 +0000 Subject: [PATCH 1/3] Add `AiCrawler` with AI-powered HTML extraction --- pyproject.toml | 3 +- src/crawlee/crawlers/__init__.py | 42 ++ src/crawlee/crawlers/_ai/__init__.py | 42 ++ src/crawlee/crawlers/_ai/_ai_crawler.py | 173 +++++ .../crawlers/_ai/_ai_crawling_context.py | 44 ++ src/crawlee/crawlers/_ai/_base_distiller.py | 66 ++ src/crawlee/crawlers/_ai/_base_extractor.py | 113 ++++ .../crawlers/_ai/_clean_html_distiller.py | 260 +++++++ src/crawlee/crawlers/_ai/_direct_extractor.py | 144 ++++ src/crawlee/crawlers/_ai/_prompts.py | 47 ++ .../crawlers/_ai/_selector_extractor.py | 633 ++++++++++++++++++ .../crawlers/_ai/_skeleton_distiller.py | 216 ++++++ src/crawlee/crawlers/_ai/_types.py | 132 ++++ src/crawlee/crawlers/_ai/_utils.py | 28 + uv.lock | 455 ++++++++++++- 15 files changed, 2395 insertions(+), 3 deletions(-) create mode 100644 src/crawlee/crawlers/_ai/__init__.py create mode 100644 src/crawlee/crawlers/_ai/_ai_crawler.py create mode 100644 src/crawlee/crawlers/_ai/_ai_crawling_context.py create mode 100644 src/crawlee/crawlers/_ai/_base_distiller.py create mode 100644 src/crawlee/crawlers/_ai/_base_extractor.py create mode 100644 src/crawlee/crawlers/_ai/_clean_html_distiller.py create mode 100644 src/crawlee/crawlers/_ai/_direct_extractor.py create mode 100644 src/crawlee/crawlers/_ai/_prompts.py create mode 100644 src/crawlee/crawlers/_ai/_selector_extractor.py create mode 100644 src/crawlee/crawlers/_ai/_skeleton_distiller.py create mode 100644 src/crawlee/crawlers/_ai/_types.py create mode 100644 src/crawlee/crawlers/_ai/_utils.py diff --git a/pyproject.toml b/pyproject.toml index 811622d742..16c43e1df4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ dependencies = [ ] [project.optional-dependencies] -all = ["crawlee[adaptive-crawler,beautifulsoup,cli,curl-impersonate,httpx,parsel,playwright,otel,sql_sqlite,sql_postgres,sql_mysql,stagehand,redis]"] +all = ["crawlee[adaptive-crawler,ai,beautifulsoup,cli,curl-impersonate,httpx,parsel,playwright,otel,sql_sqlite,sql_postgres,sql_mysql,stagehand,redis]"] adaptive-crawler = [ "jaro-winkler>=2.0.3", "playwright>=1.27.0", @@ -58,6 +58,7 @@ adaptive-crawler = [ "apify_fingerprint_datapoints>=0.0.3", "browserforge>=1.2.4" ] +ai = ["pydantic-ai-slim[openai]>=1.106.0", "parsel>=1.10.0", "lxml[html_clean]>=5.2.0"] beautifulsoup = ["beautifulsoup4[lxml]>=4.12.0", "html5lib>=1.0"] cli = ["cookiecutter>=2.6.0", "inquirer>=3.3.0", "rich>=13.9.0", "typer>=0.12.0"] curl-impersonate = ["curl-cffi>=0.9.0"] diff --git a/src/crawlee/crawlers/__init__.py b/src/crawlee/crawlers/__init__.py index ac97581bb0..2e67efa985 100644 --- a/src/crawlee/crawlers/__init__.py +++ b/src/crawlee/crawlers/__init__.py @@ -65,6 +65,36 @@ StagehandPreNavCrawlingContext, ) +with _try_import( + __name__, + 'AiCleanHtmlDistiller', + 'AiCrawler', + 'AiCrawlingContext', + 'AiDirectExtractor', + 'AiHtmlDistiller', + 'AiHtmlExtractor', + 'AiSelectorExtractor', + 'AiSkeletonDistiller', + 'AiUsageStats', + 'BaseAiHtmlDistiller', + 'BaseAiHtmlExtractor', + 'get_basic_ai_cleaner', +): + from ._ai import ( + AiCleanHtmlDistiller, + AiCrawler, + AiCrawlingContext, + AiDirectExtractor, + AiHtmlDistiller, + AiHtmlExtractor, + AiSelectorExtractor, + AiSkeletonDistiller, + AiUsageStats, + BaseAiHtmlDistiller, + BaseAiHtmlExtractor, + get_basic_ai_cleaner, + ) + __all__ = [ 'AbstractHttpCrawler', @@ -74,6 +104,17 @@ 'AdaptivePlaywrightCrawlingContext', 'AdaptivePlaywrightPostNavCrawlingContext', 'AdaptivePlaywrightPreNavCrawlingContext', + 'AiCleanHtmlDistiller', + 'AiCrawler', + 'AiCrawlingContext', + 'AiDirectExtractor', + 'AiHtmlDistiller', + 'AiHtmlExtractor', + 'AiSelectorExtractor', + 'AiSkeletonDistiller', + 'AiUsageStats', + 'BaseAiHtmlDistiller', + 'BaseAiHtmlExtractor', 'BasicCrawler', 'BasicCrawlerOptions', 'BasicCrawlingContext', @@ -99,4 +140,5 @@ 'StagehandCrawlingContext', 'StagehandPostNavCrawlingContext', 'StagehandPreNavCrawlingContext', + 'get_basic_ai_cleaner', ] diff --git a/src/crawlee/crawlers/_ai/__init__.py b/src/crawlee/crawlers/_ai/__init__.py new file mode 100644 index 0000000000..90571efc04 --- /dev/null +++ b/src/crawlee/crawlers/_ai/__init__.py @@ -0,0 +1,42 @@ +from crawlee._utils.try_import import install_import_hook as _install_import_hook +from crawlee._utils.try_import import try_import as _try_import + +_install_import_hook(__name__) + +# The following imports are wrapped in try_import to handle optional dependencies (the `ai` extra), +# ensuring the module can still function even if these dependencies are missing. +with _try_import(__name__, 'AiCrawler'): + from ._ai_crawler import AiCrawler +with _try_import(__name__, 'AiCrawlingContext'): + from ._ai_crawling_context import AiCrawlingContext +with _try_import(__name__, 'BaseAiHtmlExtractor'): + from ._base_extractor import BaseAiHtmlExtractor +with _try_import(__name__, 'AiDirectExtractor'): + from ._direct_extractor import AiDirectExtractor +with _try_import(__name__, 'AiSelectorExtractor'): + from ._selector_extractor import AiSelectorExtractor +with _try_import(__name__, 'BaseAiHtmlDistiller'): + from ._base_distiller import BaseAiHtmlDistiller +with _try_import(__name__, 'AiCleanHtmlDistiller'): + from ._clean_html_distiller import AiCleanHtmlDistiller +with _try_import(__name__, 'AiSkeletonDistiller'): + from ._skeleton_distiller import AiSkeletonDistiller +with _try_import(__name__, 'AiHtmlDistiller', 'AiHtmlExtractor', 'AiUsageStats'): + from ._types import AiHtmlDistiller, AiHtmlExtractor, AiUsageStats +with _try_import(__name__, 'get_basic_ai_cleaner'): + from ._utils import get_basic_ai_cleaner + +__all__ = [ + 'AiCleanHtmlDistiller', + 'AiCrawler', + 'AiCrawlingContext', + 'AiDirectExtractor', + 'AiHtmlDistiller', + 'AiHtmlExtractor', + 'AiSelectorExtractor', + 'AiSkeletonDistiller', + 'AiUsageStats', + 'BaseAiHtmlDistiller', + 'BaseAiHtmlExtractor', + 'get_basic_ai_cleaner', +] diff --git a/src/crawlee/crawlers/_ai/_ai_crawler.py b/src/crawlee/crawlers/_ai/_ai_crawler.py new file mode 100644 index 0000000000..5c89d20e1f --- /dev/null +++ b/src/crawlee/crawlers/_ai/_ai_crawler.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import warnings +from contextlib import AbstractAsyncContextManager +from logging import getLogger +from typing import TYPE_CHECKING + +from parsel import Selector + +from crawlee._utils.docs import docs_group +from crawlee.crawlers import AbstractHttpCrawler, HttpCrawlerOptions +from crawlee.crawlers._parsel._parsel_crawling_context import ParselCrawlingContext +from crawlee.crawlers._parsel._parsel_parser import ParselParser + +from ._ai_crawling_context import AiCrawlingContext +from ._direct_extractor import AiDirectExtractor + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + + from pydantic_ai.models import Model + from typing_extensions import Unpack + + from crawlee import Request + from crawlee.crawlers._abstract_http import ParsedHttpCrawlingContext + + from ._types import AiHtmlExtractor, AiUsageStats, ExtractFunction, TSchema + + +logger = getLogger(__name__) + + +@docs_group('Crawlers') +class AiCrawler(AbstractHttpCrawler[AiCrawlingContext, Selector, Selector]): + """A web crawler that extracts structured data from pages using an AI model. + + Builds on `AbstractHttpCrawler` and parses responses with Parsel, so the request handler has both the usual + Parsel `selector` and the AI-powered `extract` helper: pass a Pydantic model and get a validated instance back. + + The model layer is Pydantic AI, so any provider it supports (OpenAI, Anthropic, Gemini, Ollama, ...) works + through the `model` argument. The default extractor is an `AiDirectExtractor`: each page is distilled and sent + to the model in one call. For cached CSS-selector extraction at near-zero LLM cost, pass an `AiSelectorExtractor` + through the `extractor` argument. + + Warning: + This is an experimental crawler. Its public API may change in future versions. + + ### Usage + + ```python + from pydantic import BaseModel + from pydantic_ai.models.openai import OpenAIChatModel + from pydantic_ai.providers.openai import OpenAIProvider + + from crawlee.crawlers import AiCrawler, AiCrawlingContext + + + class Article(BaseModel): + title: str + author: str | None + + + crawler = AiCrawler(model=OpenAIChatModel('gpt-5.4-nano', provider=OpenAIProvider(api_key='...'))) + + + @crawler.router.default_handler + async def request_handler(context: AiCrawlingContext) -> None: + article = await context.extract(Article) + await context.push_data(article.model_dump()) + + + await crawler.run(['https://crawlee.dev/']) + ``` + """ + + def __init__( + self, + *, + model: str | Model | None = None, + extractor: AiHtmlExtractor | None = None, + **kwargs: Unpack[HttpCrawlerOptions[AiCrawlingContext]], + ) -> None: + """Initialize a new instance. + + Args: + model: The model used for extraction, given to the default extractor (`AiDirectExtractor`). A + provider-prefixed name (e.g. `'openai:gpt-5.4-nano'`) or a Pydantic AI `Model` instance. When given + as a string, the provider reads credentials from its environment variable (e.g. `OPENAI_API_KEY`). + Pass a `Model` instance to supply them explicitly. Provide exactly one of `model` or `extractor`. + extractor: A pre-configured `AiHtmlExtractor`, for full control over the distiller, instructions, + caching, usage limits, and model fallback. Pass an `AiSelectorExtractor` here for cached-selector + extraction. Provide exactly one of `model` or `extractor`. + kwargs: Additional keyword arguments to pass to the underlying `AbstractHttpCrawler`. + """ + if (model is None) == (extractor is None): + raise ValueError('Provide exactly one of `model` or `extractor`.') + + if extractor is None and model is not None: + extractor = AiDirectExtractor(model) + + if not extractor: + raise ValueError('Extractor initialization failed; check the provided model or extractor configuration.') + + # Call the notification only once. + warnings.warn( + 'The AiCrawler is experimental and its public API may change in future releases.', + category=UserWarning, + stacklevel=2, + ) + + self._ai_usage = extractor.ai_usage + self._extractor = extractor + + async def final_step( + context: ParsedHttpCrawlingContext[Selector], + ) -> AsyncGenerator[AiCrawlingContext, None]: + """Enhance `ParsedHttpCrawlingContext[Selector]` with the `extract` helper and `ai_usage`.""" + parsel_context = ParselCrawlingContext.from_parsed_http_crawling_context(context) + yield AiCrawlingContext.from_parsel_crawling_context( + parsel_context, + extract=self._create_extract_function(parsel_context.selector, parsel_context.request), + ai_usage=self._ai_usage, + ) + + kwargs['_context_pipeline'] = self._create_static_content_crawler_pipeline().compose(final_step) + + # If the extractor is an async context manager, add it to the crawler's additional context managers so it's + # properly entered and exited around the crawl. + if isinstance(extractor, AbstractAsyncContextManager): + kwargs['_additional_context_managers'] = [ + *kwargs.get('_additional_context_managers', []), + extractor, + ] + super().__init__( + parser=ParselParser(), + **kwargs, + ) + + @property + def extractor(self) -> AiHtmlExtractor: + """The extractor used to turn pages into structured data.""" + return self._extractor + + @property + def ai_usage(self) -> AiUsageStats: + """Accumulated token usage across extraction calls.""" + return self._ai_usage + + def _create_extract_function(self, selector: Selector, request: Request) -> ExtractFunction: + """Build an `extract` helper bound to the page's parsed tree. + + When the caller omits `cache_tag`, it defaults to `request.label` so an `AiSelectorExtractor` buckets + selectors per route without extra wiring. An explicit `cache_tag` overrides this. + """ + + async def extract( + schema: type[TSchema], + *, + scope: str | None = None, + cache_tag: str | None = None, + additional_instructions: str | None = None, + ) -> TSchema: + # `AiHtmlExtractor.extract` accepts a Selector directly, so the already-parsed tree is handed over + # without a serialize round trip. + return await self._extractor.extract( + selector, + schema, + scope=scope, + cache_tag=cache_tag if cache_tag is not None else request.label, + additional_instructions=additional_instructions, + ) + + return extract diff --git a/src/crawlee/crawlers/_ai/_ai_crawling_context.py b/src/crawlee/crawlers/_ai/_ai_crawling_context.py new file mode 100644 index 0000000000..18377a6644 --- /dev/null +++ b/src/crawlee/crawlers/_ai/_ai_crawling_context.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from dataclasses import dataclass, fields +from typing import TYPE_CHECKING + +from crawlee._utils.docs import docs_group +from crawlee.crawlers._parsel._parsel_crawling_context import ParselCrawlingContext + +if TYPE_CHECKING: + from typing_extensions import Self + + from ._types import AiUsageStats, ExtractFunction + + +@dataclass(frozen=True) +@docs_group('Crawling contexts') +class AiCrawlingContext(ParselCrawlingContext): + """The crawling context used by the `AiCrawler`. + + It extends `ParselCrawlingContext`, so the full Parsel `selector` (and `enqueue_links`) remain available + alongside the AI-powered `extract` helper. Handlers can mix cheap manual selectors with AI extraction on the + same page. + """ + + extract: ExtractFunction + """Extract a structured Pydantic model from the page using the configured AI extractor.""" + + ai_usage: AiUsageStats + """The cumulative token usage stats of the extractor across calls in this crawl.""" + + @classmethod + def from_parsel_crawling_context( + cls, + context: ParselCrawlingContext, + *, + extract: ExtractFunction, + ai_usage: AiUsageStats, + ) -> Self: + """Create a new context from an existing `ParselCrawlingContext`.""" + return cls( + extract=extract, + ai_usage=ai_usage, + **{field.name: getattr(context, field.name) for field in fields(context)}, + ) diff --git a/src/crawlee/crawlers/_ai/_base_distiller.py b/src/crawlee/crawlers/_ai/_base_distiller.py new file mode 100644 index 0000000000..3567054167 --- /dev/null +++ b/src/crawlee/crawlers/_ai/_base_distiller.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import re +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +from crawlee._utils.docs import docs_group + +if TYPE_CHECKING: + from lxml.html import HtmlElement + + +# Placeholder tag used to hide JSON scripts from the cleaning pass. The cleaner removes `

Item

' + ) + + assert distilled_html == '

Item

' + + +def test_drops_noise_tags() -> None: + distilled_html = AiCleanHtmlDistiller().distill('

Item

') + + assert distilled_html == '

Item

' + + +def test_saves_json_ld_script() -> None: + html = '
' + distilled_html = AiCleanHtmlDistiller().distill(html) + + assert distilled_html == html + + +def test_drops_data_uri_attribute() -> None: + distilled_html = AiCleanHtmlDistiller().distill('logo') + + assert distilled_html == 'logo' + + +def test_limited_class_attribute() -> None: + distilled_html = AiCleanHtmlDistiller(max_classes=2).distill('
x
') + + assert distilled_html == '
x
' + + +def test_drops_empty_class_attribute() -> None: + distilled_html = AiCleanHtmlDistiller().distill('
x
') + + assert distilled_html == '
x
' + + +def test_truncates_long_attribute_values() -> None: + distilled_html = AiCleanHtmlDistiller(max_attr_len=5).distill(f'link') + + assert distilled_html == f'link' + + +def test_truncates_json_payload() -> None: + distilled_html = AiCleanHtmlDistiller(max_json_len=5).distill( + '
' + ) + + assert distilled_html == f'
' + + +def test_keep_head_useful_tags() -> None: + html = ( + '' + 'Page' + '' + '' + '' + '' + '

Body

' + ) + distilled_html = AiCleanHtmlDistiller().distill(html) + + assert distilled_html == ( + '' + 'Page' + '' + '' + '

Body

' + ) + + +def test_drops_head() -> None: + distilled_html = AiCleanHtmlDistiller(keep_head=False).distill( + 'Page

Body

' + ) + + assert distilled_html == '

Body

' + + +def test_normalizes_whitespace() -> None: + distilled_html = AiCleanHtmlDistiller().distill('

a b\n\n\tc

') + + assert distilled_html == '

a b c

' + + +def test_enforces_max_size(caplog: pytest.LogCaptureFixture) -> None: + html = f'
{"

x

" * 500}
' + with caplog.at_level(logging.WARNING, logger='crawlee.crawlers._ai._clean_html_distiller'): + out = AiCleanHtmlDistiller(max_size=50).distill(html) + + assert len(out) == 50 + len(_TRUNCATION_MARKER) + assert out.endswith(_TRUNCATION_MARKER) + assert any('max_size' in record.message for record in caplog.records) diff --git a/tests/unit/crawlers/_ai/test_direct_extractor.py b/tests/unit/crawlers/_ai/test_direct_extractor.py new file mode 100644 index 0000000000..3fb2495b4b --- /dev/null +++ b/tests/unit/crawlers/_ai/test_direct_extractor.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +from parsel import Selector +from pydantic import BaseModel +from pydantic_ai import capture_run_messages +from pydantic_ai.exceptions import UnexpectedModelBehavior +from pydantic_ai.messages import ModelRequest, UserPromptPart +from pydantic_ai.models.test import TestModel + +from crawlee.crawlers import AiDirectExtractor, BaseAiHtmlDistiller +from crawlee.crawlers._ai._prompts import _DIRECT_INSTRUCTIONS + +if TYPE_CHECKING: + from pydantic_ai.messages import ModelMessage + + +DEFAULT_OUTPUT_ARGS = {'name': 'Phone', 'price': '$9'} + + +class _Product(BaseModel): + name: str + price: str | None = None + + +class _MockDistiller(BaseAiHtmlDistiller): + """Distiller returning a fixed marker so the prompt content can be asserted.""" + + def distill(self, html: str) -> str: + _html = html + return 'MOCK-DISTILLED-HTML' + + def get_prompt_notes(self) -> str | None: + return 'MOCK-NOTES' + + +def _extract_model_input(messages: list[ModelMessage]) -> tuple[str, str]: + """Return the (user prompt, instructions) of the prompt request the model received. + + A run produces several requests (the prompt, then a tool-return follow-up). Only the first carries the user + prompt, so it is the one to inspect. + """ + request = next( + message + for message in messages + if isinstance(message, ModelRequest) and any(isinstance(part, UserPromptPart) for part in message.parts) + ) + prompt = next(part.content for part in request.parts if isinstance(part, UserPromptPart)) + return str(prompt), request.instructions or '' + + +async def test_returns_validated_model() -> None: + extractor = AiDirectExtractor(TestModel(custom_output_args=DEFAULT_OUTPUT_ARGS)) + + result = await extractor.extract('', _Product) + + assert isinstance(result, _Product) + assert result.name == 'Phone' + assert result.price == '$9' + + +async def test_counts_token_usage() -> None: + extractor = AiDirectExtractor(TestModel(custom_output_args=DEFAULT_OUTPUT_ARGS)) + + await extractor.extract('', _Product) + + assert extractor.ai_usage.requests == 1 + assert extractor.ai_usage.input_tokens > 0 + assert extractor.ai_usage.output_tokens > 0 + assert extractor.ai_usage.total_tokens > 0 + + +async def test_accepts_selector_input() -> None: + extractor = AiDirectExtractor(TestModel(custom_output_args=DEFAULT_OUTPUT_ARGS)) + + html = '
UNIQUE-CONTENT
' + with capture_run_messages() as messages: + await extractor.extract(Selector(text=html), _Product) + + prompt, _ = _extract_model_input(messages) + + html_part = prompt.split('Document:')[1].strip() + + assert html_part == html + + +async def test_scope_subtree() -> None: + extractor = AiDirectExtractor(TestModel(custom_output_args=DEFAULT_OUTPUT_ARGS)) + + with capture_run_messages() as messages: + await extractor.extract( + '

Phone

junk
', + _Product, + scope='article', + ) + + prompt, _ = _extract_model_input(messages) + + html_part = prompt.split('Document:')[1].strip() + + assert html_part == '

Phone

' + + +async def test_scope_raises() -> None: + extractor = AiDirectExtractor(TestModel(custom_output_args=DEFAULT_OUTPUT_ARGS)) + + with pytest.raises(ValueError, match='matched nothing'): + await extractor.extract('
x
', _Product, scope='.missing') + + +async def test_input_prompt() -> None: + extractor = AiDirectExtractor(TestModel(custom_output_args=DEFAULT_OUTPUT_ARGS), distiller=_MockDistiller()) + + with capture_run_messages() as messages: + await extractor.extract('', _Product) + + prompt, _ = _extract_model_input(messages) + + assert 'name, price' in prompt + assert 'MOCK-DISTILLED-HTML' in prompt + + +async def test_instructions() -> None: + extractor = AiDirectExtractor( + TestModel(custom_output_args=DEFAULT_OUTPUT_ARGS), + distiller=_MockDistiller(), + ) + + with capture_run_messages() as messages: + await extractor.extract('', _Product) + + _, instructions = _extract_model_input(messages) + + assert 'MOCK-NOTES' in instructions + assert _DIRECT_INSTRUCTIONS in instructions + + +async def test_additional_instructions() -> None: + extractor = AiDirectExtractor(TestModel(custom_output_args=DEFAULT_OUTPUT_ARGS), distiller=_MockDistiller()) + + with capture_run_messages() as messages: + await extractor.extract('

Phone

', _Product, additional_instructions='PER-CALL-HINT') + + _, instructions = _extract_model_input(messages) + + # Both the base instructions and the per-call hint reach the model. + assert 'PER-CALL-HINT' in instructions + assert _DIRECT_INSTRUCTIONS in instructions + + +async def test_raise_for_invalid_output() -> None: + # `name` is required, so output missing it fails validation on every retry until the run errors. + extractor = AiDirectExtractor(TestModel(custom_output_args={'price': '$9'}), retries=2) + + with pytest.raises(UnexpectedModelBehavior): + await extractor.extract('

x

', _Product) + + # The extractor's usage stats reflect the 3 failed attempts (1 initial + 2 retries). + assert extractor.ai_usage.requests == 3 diff --git a/tests/unit/crawlers/_ai/test_selector_extractor.py b/tests/unit/crawlers/_ai/test_selector_extractor.py new file mode 100644 index 0000000000..d95c0f9210 --- /dev/null +++ b/tests/unit/crawlers/_ai/test_selector_extractor.py @@ -0,0 +1,343 @@ +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING, Literal + +import pytest +from pydantic import BaseModel, create_model +from pydantic_ai.exceptions import UnexpectedModelBehavior +from pydantic_ai.messages import ModelResponse, ToolCallPart +from pydantic_ai.models.function import FunctionModel +from pydantic_ai.models.test import TestModel + +from crawlee.crawlers import AiDirectExtractor, AiSelectorExtractor, AiUsageStats + +if TYPE_CHECKING: + from typing import Any + + from pydantic_ai.messages import ModelMessage + from pydantic_ai.models import Model + from pydantic_ai.models.function import AgentInfo + + from crawlee.crawlers._ai._types import AiHtmlExtractor + + +class _Item(BaseModel): + name: str + + +class _Posts(BaseModel): + posts: list[_Item] + + +class _Nested(BaseModel): + title: str + item: _Item | None = None + + +class _Collections(BaseModel): + items: list[str] + unique: set[str] + + +class _Status(BaseModel): + status: Literal['in_stock', 'sold_out'] + + +class _Mapping(BaseModel): + data: dict[str, str] + + +NAME_HTML = '
X
' +LIST_HTML = '' + +NAME_SELECTORS = {'selectors': {'name': {'selector': '.n::text'}}} +POSTS_SELECTORS = {'selectors': {'posts': {'selector': 'li.r', 'fields': {'name': {'selector': '.t::text'}}}}} + + +def _model(*plans: dict[str, Any]) -> FunctionModel: + """Build a model that returns the given selector maps in order, repeating the last for any further calls.""" + state = {'index': 0} + + def respond(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + _messages = messages + plan = plans[min(state['index'], len(plans) - 1)] + state['index'] += 1 + return ModelResponse(parts=[ToolCallPart(tool_name=info.output_tools[0].name, args=plan)]) + + return FunctionModel(respond) + + +def _extractor( + model: str | Model | None = None, + *, + fallback: AiHtmlExtractor | None = None, + retries: int = 3, + max_variants: int = 5, +) -> AiSelectorExtractor: + return AiSelectorExtractor( + model or TestModel(), + persistence=False, + fallback=fallback, + retries=retries, + max_variants=max_variants, + ) + + +@pytest.mark.parametrize( + ('annotation', 'match'), + [ + pytest.param(dict[str, str], 'mapping', id='dict'), + pytest.param(list[int | str], 'list of a union', id='list-of-union'), + pytest.param(list[list[str]], 'list of lists', id='list-of-lists'), + pytest.param(tuple[int, ...], 'unsupported annotation', id='tuple'), + pytest.param(int | str, 'unsupported annotation', id='scalar-union'), + pytest.param(_Posts, 'deeper than one level', id='deep-nesting'), + ], +) +async def test_rejects_unsupported_schema(annotation: Any, match: str) -> None: + # The schema shape is checked before any model call, so an unsupported one raises with the reason. + schema = create_model('_Unsupported', field=(annotation, ...)) + + with pytest.raises(ValueError, match=match): + await _extractor().extract('
', schema) + + +async def test_extracts_scalar() -> None: + result = await _extractor(_model(NAME_SELECTORS)).extract(NAME_HTML, _Item) + + assert result == _Item(name='X') + + +async def test_extracts_collection_of_scalars() -> None: + plan = {'selectors': {'items': {'selector': '.a::text'}, 'unique': {'selector': '.b::text'}}} + html = '
xypp
' + + result = await _extractor(_model(plan)).extract(html, _Collections) + + # The list keeps order and duplicates; the set is deduplicated. + assert result == _Collections(items=['x', 'y'], unique={'p'}) + + +async def test_extracts_list_of_items() -> None: + result = await _extractor(_model(POSTS_SELECTORS)).extract(LIST_HTML, _Posts) + + assert result == _Posts(posts=[_Item(name='A'), _Item(name='B')]) + + +async def test_extracts_literal_field() -> None: + plan = {'selectors': {'status': {'selector': '.s::text'}}} + + result = await _extractor(_model(plan)).extract('
in_stock
', _Status) + + assert result == _Status(status='in_stock') + + +async def test_reuses_cached_plan() -> None: + extractor = _extractor(_model(NAME_SELECTORS)) + + assert await extractor.extract(NAME_HTML, _Item, cache_tag='test') == _Item(name='X') + assert await extractor.extract(NAME_HTML, _Item, cache_tag='test') == _Item(name='X') + # The second extract is served from the cache, so the model was consulted only once. + assert extractor.ai_usage.requests == 1 + + +async def test_concurrent_generate() -> None: + extractor = _extractor(_model(NAME_SELECTORS)) + + results = await asyncio.gather( + extractor.extract(NAME_HTML, _Item, cache_tag='test'), + extractor.extract(NAME_HTML, _Item, cache_tag='test'), + ) + + assert results == [_Item(name='X'), _Item(name='X')] + assert extractor.ai_usage.requests == 1 + + +async def test_cached_plan_for_optional_field() -> None: + plan = { + 'selectors': { + 'title': {'selector': 'h1::text'}, + 'item': {'selector': '.item', 'fields': {'name': {'selector': '.n::text'}}}, + } + } + extractor = _extractor(_model(plan)) + + first_call = await extractor.extract( + '

T

X
', _Nested, cache_tag='test' + ) + # The second page has no item, but the cached plan is still valid and returns None for the optional field. + second_call = await extractor.extract('

T2

', _Nested, cache_tag='test') + + assert first_call == _Nested(title='T', item=_Item(name='X')) + assert second_call == _Nested(title='T2', item=None) + assert extractor.ai_usage.requests == 1 + + +async def test_caches_different_tags() -> None: + extractor = _extractor(_model(NAME_SELECTORS)) + + await extractor.extract(NAME_HTML, _Item, cache_tag='a') + await extractor.extract(NAME_HTML, _Item, cache_tag='b') # different bucket, generated again + await extractor.extract(NAME_HTML, _Item, cache_tag='a') # first bucket still cached + + # One generation per tag. A shared bucket would have served 'b' from cache, leaving the count at one. + assert extractor.ai_usage.requests == 2 + + +async def test_eviction_of_oldest_variant() -> None: + pages = { + 'a': '
A
', + 'b': '
B
', + 'c': '
C
', + } + extractor = _extractor( + _model( + {'selectors': {'name': {'selector': '.a::text'}}}, + {'selectors': {'name': {'selector': '.b::text'}}}, + {'selectors': {'name': {'selector': '.c::text'}}}, # 'c' generated and evicted 'a' + {'selectors': {'name': {'selector': '.a::text'}}}, # 'a' regenerated after eviction + ), + max_variants=2, + ) + + for key in ('a', 'b', 'c'): + await extractor.extract(pages[key], _Item, cache_tag='test') + + # With max_variants=2 the 'a' plan was evicted, so extracting 'a' again generates a fourth time. + await extractor.extract(pages['a'], _Item, cache_tag='test') + + assert extractor.ai_usage.requests == 4 + + +@pytest.mark.parametrize( + 'invalid_plan', + [ + pytest.param({'selectors': {}}, id='missing-field'), + pytest.param( + {'selectors': {'posts': {'selector': 'li.r::text', 'fields': {'name': {'selector': '.t::text'}}}}}, + id='container-with-value-form', + ), + pytest.param( + {'selectors': {'posts': {'selector': 'li[', 'fields': {'name': {'selector': '.t::text'}}}}}, + id='invalid-css', + ), + pytest.param( + {'selectors': {'posts': {'selector': 'li.r', 'fields': {'name': {'selector': '.t'}}}}}, + id='sub-without-value-form', + ), + pytest.param( + {'selectors': {'posts': {'selector': 'li.r', 'fields': {'name': {'selector': '.absent::text'}}}}}, + id='sub-matches-nothing', + ), + ], +) +async def test_retries_with_invalid_plan(invalid_plan: dict[str, Any]) -> None: + # Each plan trips a different validation guard. The model is asked to fix it, then returns a valid plan. + extractor = _extractor(_model(invalid_plan, POSTS_SELECTORS)) + + result = await extractor.extract(LIST_HTML, _Posts, cache_tag='test') + + assert result == _Posts(posts=[_Item(name='A'), _Item(name='B')]) + assert extractor.ai_usage.requests == 2 # the first plan failed validation, the second one succeeded + + +async def test_retries_with_invalid_data() -> None: + # The selectors are well-formed and match, but the extracted value fails schema validation, so the plan is retried. + html = '
WRONGin_stock
' + extractor = _extractor( + _model( + {'selectors': {'status': {'selector': '.bad::text'}}}, # 'WRONG' is not a valid Literal value + {'selectors': {'status': {'selector': '.good::text'}}}, + ) + ) + + result = await extractor.extract(html, _Status, cache_tag='test') + + assert result == _Status(status='in_stock') + assert extractor.ai_usage.requests == 2 + + +async def test_unsupported_schema_delegates_to_fallback() -> None: + fallback = AiDirectExtractor(TestModel(custom_output_args={'data': {'k': 'v'}})) + + result = await _extractor(fallback=fallback).extract('
', _Mapping) + + assert result == _Mapping(data={'k': 'v'}) + + +async def test_generation_failure_delegates_to_fallback() -> None: + # The selector matches nothing, so generation fails and the extractor degrades to the fallback. + bad_plan = {'selectors': {'name': {'selector': '.absent::text'}}} + fallback = AiDirectExtractor(TestModel(custom_output_args={'name': 'from-fallback'})) + + result = await _extractor(_model(bad_plan), fallback=fallback, retries=0).extract( + NAME_HTML, _Item, cache_tag='test' + ) + + assert result == _Item(name='from-fallback') + + +async def test_generation_failure_raises() -> None: + bad_plan = {'selectors': {'name': {'selector': '.absent::text'}}} + + with pytest.raises(UnexpectedModelBehavior): + await _extractor(_model(bad_plan), retries=0).extract(NAME_HTML, _Item, cache_tag='test') + + +async def test_scope_raises() -> None: + with pytest.raises(ValueError, match='matched nothing'): + await _extractor().extract('
x
', _Item, scope='.missing') + + +async def test_fallback_shares_usage_accumulator() -> None: + fallback = AiDirectExtractor(TestModel()) + extractor = _extractor(fallback=fallback) + + assert fallback.ai_usage is extractor.ai_usage + + +def test_set_ai_usage_reshares_with_fallback() -> None: + fallback = AiDirectExtractor(TestModel()) + extractor = _extractor(fallback=fallback) + new_usage = AiUsageStats() + + extractor.set_ai_usage(new_usage) + + assert extractor.ai_usage is new_usage + assert fallback.ai_usage is new_usage + + +async def test_active_state() -> None: + extractor = _extractor() + + assert extractor.active is False + async with extractor: + assert extractor.active is True + assert extractor.active is False + + +async def test_double_enter_raises() -> None: + extractor = _extractor() + + async with extractor: + with pytest.raises(RuntimeError, match='already active'): + await extractor.__aenter__() + + +async def test_exit_without_enter_raises() -> None: + extractor = _extractor() + + with pytest.raises(RuntimeError, match='not active'): + await extractor.__aexit__(None, None, None) + + +async def test_cache_persists_across_instances() -> None: + async with AiSelectorExtractor(_model(NAME_SELECTORS), kvs_cache_key='shared-cache') as first: + assert await first.extract(NAME_HTML, _Item, cache_tag='test') == _Item(name='X') + assert first.ai_usage.requests == 1 + + # A fresh instance loads the cache from the KeyValueStore and serves without calling the model. + async with AiSelectorExtractor(_model(NAME_SELECTORS), kvs_cache_key='shared-cache') as second: + assert await second.extract(NAME_HTML, _Item, cache_tag='test') == _Item(name='X') + assert second.ai_usage.requests == 0 diff --git a/tests/unit/crawlers/_ai/test_skeleton_distiller.py b/tests/unit/crawlers/_ai/test_skeleton_distiller.py new file mode 100644 index 0000000000..30a2bc89c3 --- /dev/null +++ b/tests/unit/crawlers/_ai/test_skeleton_distiller.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from crawlee.crawlers import AiSkeletonDistiller +from crawlee.crawlers._ai._prompts import _SKELETON_PROMPT_NOTES, _TRUNCATION_MARKER + +if TYPE_CHECKING: + import pytest + + +def test_default_prompt_notes() -> None: + assert AiSkeletonDistiller().get_prompt_notes() == _SKELETON_PROMPT_NOTES + + +def test_truncates_long_text() -> None: + distilled_html = AiSkeletonDistiller(max_text_len=5).distill(f'

{"a" * 20}

') + + assert distilled_html == f'

aaaaa{_TRUNCATION_MARKER}

' + + +def test_keeps_short_text() -> None: + distilled_html = AiSkeletonDistiller(max_text_len=50).distill('

short text

') + + assert distilled_html == '

short text

' + + +def test_collapses_repeated_siblings() -> None: + items = ''.join(f'
  • item {index}
    • ' for index in range(10)) + distilled_html = AiSkeletonDistiller(keep_siblings=3).distill(f'') + + assert distilled_html == ( + '' + ) + + +def test_does_not_collapse_siblings_with_different_identity_attrs() -> None: + # Same tag and class, but different identity attributes, not a repeating template. + spans = ''.join(f'v' for index in range(4)) + distilled_html = AiSkeletonDistiller(keep_siblings=2).distill(f'
    {spans}
    ') + + assert distilled_html == ( + '
    ' + 'v' + 'v' + 'v' + 'v' + '
    ' + ) + + +def test_does_not_collapse_scripts() -> None: + scripts = '' * 4 + distilled_html = AiSkeletonDistiller(keep_siblings=2).distill(f'
    {scripts}
    ') + + assert distilled_html == ( + '
    ' + '' + '' + '' + '' + '
    ' + ) + + +def test_does_not_collapse_layout_markers() -> None: + distilled_html = AiSkeletonDistiller(keep_siblings=2).distill(f'
    {"
    " * 5}
    ') + + assert distilled_html == '





    ' + + +def test_redistills_for_oversize_without_cutting(caplog: pytest.LogCaptureFixture) -> None: + text = 'a' * 50 + html = f'

    {text}

    {text}

    {text}

    ' + + with caplog.at_level(logging.WARNING, logger='crawlee.crawlers._ai._skeleton_distiller'): + distilled_html = AiSkeletonDistiller(max_text_len=20, max_size=120).distill(html) + + # The first distillation produces a skeleton of 134 chars, but the limit is 120 chars. + # The second distillation uses more aggressive text truncation to 15 chars, so the result is 119 chars. + assert distilled_html == ( + f'
    ' + f'

    aaaaaaaaaaaaaaa{_TRUNCATION_MARKER}

    ' + f'

    aaaaaaaaaaaaaaa{_TRUNCATION_MARKER}

    ' + f'

    aaaaaaaaaaaaaaa{_TRUNCATION_MARKER}

    ' + f'
    ' + ) + assert not caplog.records + + +def test_cutting_for_oversize(caplog: pytest.LogCaptureFixture) -> None: + text = 'a' * 50 + html = f'

    {text}

    {text}

    {text}

    ' + + with caplog.at_level(logging.WARNING, logger='crawlee.crawlers._ai._skeleton_distiller'): + distilled_html = AiSkeletonDistiller(max_text_len=20, max_size=32).distill(html) + + assert distilled_html == f'

    aaaaaaaaaaaaaaa{_TRUNCATION_MARKER}' + assert any('max_size' in record.message for record in caplog.records) From 7f496a686e0150595c89f5aeba4153a9d738b9e6 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Wed, 17 Jun 2026 18:51:00 +0000 Subject: [PATCH 3/3] add docs --- docs/guides/ai_crawler.mdx | 150 ++++++++++++++++++ docs/guides/architecture_overview.mdx | 11 +- .../additional_instructions_example.py | 44 +++++ .../code_examples/ai_crawler/basic_example.py | 41 +++++ .../ai_crawler/custom_distiller_example.py | 67 ++++++++ .../ai_crawler/selector_extractor_example.py | 56 +++++++ .../ai_crawler/usage_limit_example.py | 57 +++++++ src/crawlee/crawlers/_ai/_ai_crawler.py | 19 +-- tests/unit/crawlers/_ai/test_ai_crawler.py | 15 +- 9 files changed, 445 insertions(+), 15 deletions(-) create mode 100644 docs/guides/ai_crawler.mdx create mode 100644 docs/guides/code_examples/ai_crawler/additional_instructions_example.py create mode 100644 docs/guides/code_examples/ai_crawler/basic_example.py create mode 100644 docs/guides/code_examples/ai_crawler/custom_distiller_example.py create mode 100644 docs/guides/code_examples/ai_crawler/selector_extractor_example.py create mode 100644 docs/guides/code_examples/ai_crawler/usage_limit_example.py diff --git a/docs/guides/ai_crawler.mdx b/docs/guides/ai_crawler.mdx new file mode 100644 index 0000000000..17d030fe89 --- /dev/null +++ b/docs/guides/ai_crawler.mdx @@ -0,0 +1,150 @@ +--- +id: ai-crawler +title: AI crawler +description: Learn how to use AiCrawler to extract structured data from HTML pages with an LLM. +--- + +import ApiLink from '@site/src/components/ApiLink'; +import CodeBlock from '@theme/CodeBlock'; + +import BasicExample from '!!raw-loader!./code_examples/ai_crawler/basic_example.py'; +import AdditionalInstructionsExample from '!!raw-loader!./code_examples/ai_crawler/additional_instructions_example.py'; +import CustomDistillerExample from '!!raw-loader!./code_examples/ai_crawler/custom_distiller_example.py'; +import SelectorExtractorExample from '!!raw-loader!./code_examples/ai_crawler/selector_extractor_example.py'; +import UsageLimitExample from '!!raw-loader!./code_examples/ai_crawler/usage_limit_example.py'; + +An `AiCrawler` extracts structured data from a page with an LLM. It fetches each page over plain HTTP and parses it with Parsel, then exposes an `extract` helper: pass a Pydantic model and get a validated instance back. Instead of writing CSS selectors for every field, you describe the data with a schema and the model fills it in. + +The model layer is [Pydantic AI](https://ai.pydantic.dev/), so any provider it supports (OpenAI, Anthropic, Gemini, Ollama, ...) works through the `model` argument. The context is an `AiCrawlingContext`, which extends the `ParselCrawlingContext`, so the manual `selector` and `enqueue_links` stay available next to `extract`. + +:::caution Experimental + +`AiCrawler` is experimental. Its public API may change in future releases. + +::: + +## When to use AiCrawler + +Use `AiCrawler` when: + +- Selectors are unknown or brittle. The model reads the content, so it tolerates markup that varies or changes. +- One schema spans many layouts. A single Pydantic model fits differently structured pages, with no per-page selectors. +- Rapid prototyping. You describe the data with a schema instead of writing selectors. + +For pages with a stable, known structure, a plain `ParselCrawler` or `BeautifulSoupCrawler` is cheaper, since it runs no model calls. + +`AiCrawler` fetches pages over plain HTTP and does not render JavaScript. For pages that need a browser, or for complex multi-step interactions, use `StagehandCrawler`. See the [Stagehand crawler guide](./stagehand-crawler). + +## Installation + +`AiCrawler` requires the `ai` optional dependency group: + +```bash +pip install 'crawlee[ai]' +``` + +or with uv: + +```bash +uv add 'crawlee[ai]' +``` + +The `ai` extra installs the OpenAI integration by default. To use another provider, add the matching [pydantic-ai-slim](https://ai.pydantic.dev/install/#use-with-pydantic-ai-slim) extra. For example, for Anthropic: + +```bash +pip install 'crawlee[ai]' 'pydantic-ai-slim[anthropic]' +``` + +## Basic usage + +Provide a `model` and call `context.extract` with a Pydantic model inside the handler. The example below extracts an article and pushes it to the dataset. + + + {BasicExample} + + +The `model` builds the crawler's default extractor, an `AiDirectExtractor`. With neither `model` nor `extractor`, a default OpenAI model is used. + +The `model` argument accepts a provider-prefixed name or a Pydantic AI `Model` instance. + +```python +# A provider-prefixed name reads credentials from the provider's environment variable (e.g. OPENAI_API_KEY). +crawler = AiCrawler(model='openai:gpt-5.4-nano') + +# A Model instance takes credentials explicitly. +from pydantic_ai.models.openai import OpenAIChatModel +from pydantic_ai.providers.openai import OpenAIProvider + +model = OpenAIChatModel('gpt-5.4-nano', provider=OpenAIProvider(api_key='...')) +crawler = AiCrawler(model=model) +``` + +## Extractors + +An extractor turns a page into your schema. Extractors implement different strategies for working with the LLM, and each one uses an `AiHtmlDistiller` to shape the model's input. Crawlee ships two. + +### AiDirectExtractor + +`AiDirectExtractor` sends the distilled page to the model in one call. The schema is the model's output type. Pydantic AI validates the result; on a mismatch, it sends the error back to the model to fix, bounded by `retries`. + +It reads each page on its own, so extraction is accurate per page. It accepts schemas of any shape: nested models, lists, dictionaries, unions, and deep nesting. The cost is one model call per page, which scales poorly on a large site. + +Use `additional_instructions` to focus the model on the data you want: + + + {AdditionalInstructionsExample} + + +### AiSelectorExtractor + +`AiSelectorExtractor` asks the model for reusable CSS selectors on the first page of a route, caches them, and reuses them with no model call on later pages of the same layout, so it scales to large sites. When a page matches none of the cached selectors (a different markup variant), it generates and caches a new set, so one bucket can hold several variants. If selector generation fails, or the schema shape is unsupported, it degrades to the `fallback` extractor when one is set, and raises otherwise. Selectors are bucketed by `cache_tag`, which defaults to the request label, so each route keeps its own set. The cache is persisted to a `KeyValueStore`, so a later run reuses selectors learned earlier. + + + {SelectorExtractorExample} + + +It supports schemas built from scalar fields, lists of scalars, lists of items, and a single nested item, one level deep. For shapes it cannot serve (such as a `dict` field), set a `fallback` or use `AiDirectExtractor`. + +Both extractors share two more knobs. `retries` caps how many times the model may fix output that fails schema validation (default 1 for `AiDirectExtractor`, 3 for `AiSelectorExtractor`). `instructions` replaces the base task instructions entirely. + +## Distillers + +A distiller reduces raw HTML to a compact representation the model reads cheaply. Each extractor uses one. Replace it with the extractor's `distiller` argument (the crawler itself has no `distiller` argument). + +`AiDirectExtractor` defaults to an `AiCleanHtmlDistiller`: cleaned, structure-preserving HTML that keeps the full page text. `AiSelectorExtractor` uses an `AiSkeletonDistiller` internally to ask the model for selectors; you rarely set it yourself. + +### Custom distiller + +Subclass `BaseAiHtmlDistiller` and implement `distill` to send a different representation. Set `prompt_notes` so the model knows the input format. The extractor appends the notes to its instructions. + +The example below converts the cleaned page to Markdown with [html-to-markdown](https://pypi.org/project/html-to-markdown/), an extra dependency: + +```bash +pip install html-to-markdown +``` + + + {CustomDistillerExample} + + +## Extract options + +`context.extract` takes options alongside the schema: + +- `scope` - a CSS selector that restricts extraction to the first matching subtree (e.g. `main` or `article.post`). It saves tokens and keeps the model away from unrelated parts of the page. +- `cache_tag` - the bucket for cached selectors. It defaults to the request label. +- `additional_instructions` - extra instructions for this call, appended to the base instructions. With `AiSelectorExtractor` they steer the one-time selector generation, not each extraction, so use them to point the model at the right region. + +## Usage and cost + +Token usage accumulates on `context.ai_usage`, and on `crawler.ai_usage` for the whole crawl. The accumulator is an `AiUsageStats` with `requests`, `input_tokens`, `output_tokens`, and `total_tokens`. + +To cap spend, pass `usage_limits` (a pydantic-ai `UsageLimits`) to an extractor. It applies to every model run, and `extract` raises `UsageLimitExceeded` when a page needs more. The example below caps each extraction, logs and skips pages that exceed it, and stops the whole crawl once a token budget is spent. + + + {UsageLimitExample} + + +## Conclusion + +This guide introduced `AiCrawler` and its `extract` helper, the `AiDirectExtractor` and `AiSelectorExtractor` strategies, the built-in and custom distillers, the extract options, and how failures and cost are handled. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/docs/guides/architecture_overview.mdx b/docs/guides/architecture_overview.mdx index f9c4b764fb..f86f5041da 100644 --- a/docs/guides/architecture_overview.mdx +++ b/docs/guides/architecture_overview.mdx @@ -49,6 +49,8 @@ class ParselCrawler class BeautifulSoupCrawler +class AiCrawler + class PlaywrightCrawler class AdaptivePlaywrightCrawler @@ -65,6 +67,7 @@ BasicCrawler --|> AdaptivePlaywrightCrawler AbstractHttpCrawler --|> HttpCrawler AbstractHttpCrawler --|> ParselCrawler AbstractHttpCrawler --|> BeautifulSoupCrawler +AbstractHttpCrawler --|> AiCrawler PlaywrightCrawler --|> StagehandCrawler ``` @@ -72,11 +75,12 @@ PlaywrightCrawler --|> StagehandCrawler HTTP crawlers use HTTP clients to fetch pages and parse them with HTML parsing libraries. They are fast and efficient for sites that do not require JavaScript rendering. HTTP clients are Crawlee components that wrap around HTTP libraries like [httpx](https://www.python-httpx.org/), [curl-impersonate](https://github.com/lwthiker/curl-impersonate) or [impit](https://apify.github.io/impit) and handle HTTP communication for requests and responses. You can learn more about them in the [HTTP clients guide](./http-clients). -HTTP crawlers inherit from `AbstractHttpCrawler` and there are three crawlers that belong to this category: +HTTP crawlers inherit from `AbstractHttpCrawler` and there are four crawlers that belong to this category: - `BeautifulSoupCrawler` utilizes the [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) HTML parser. - `ParselCrawler` utilizes [Parsel](https://github.com/scrapy/parsel) for parsing HTML. - `HttpCrawler` does not parse HTTP responses at all and is used when no content parsing is required. +- `AiCrawler` parses HTML with Parsel and uses an LLM to extract structured data into a validated Pydantic model. You can learn more about HTTP crawlers in the [HTTP crawlers guide](./http-crawlers). @@ -120,6 +124,8 @@ class ParselCrawlingContext class BeautifulSoupCrawlingContext +class AiCrawlingContext + class PlaywrightPreNavCrawlingContext class PlaywrightCrawlingContext @@ -148,6 +154,8 @@ ParsedHttpCrawlingContext --|> ParselCrawlingContext ParsedHttpCrawlingContext --|> BeautifulSoupCrawlingContext +ParselCrawlingContext --|> AiCrawlingContext + BasicCrawlingContext --|> PlaywrightPreNavCrawlingContext PlaywrightPreNavCrawlingContext --|> PlaywrightCrawlingContext @@ -168,6 +176,7 @@ They have a similar inheritance structure as the crawlers, with the base class b - `ParsedHttpCrawlingContext` for HTTP crawlers with parsed responses. - `ParselCrawlingContext` for HTTP crawlers that use [Parsel](https://github.com/scrapy/parsel) for parsing. - `BeautifulSoupCrawlingContext` for HTTP crawlers that use [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) for parsing. +- `AiCrawlingContext` for the AI crawler, extending the Parsel context with an `extract` helper. - `PlaywrightPreNavCrawlingContext` for Playwright crawlers before the page is navigated. - `PlaywrightCrawlingContext` for Playwright crawlers. - `AdaptivePlaywrightPreNavCrawlingContext` for Adaptive Playwright crawlers before the page is navigated. diff --git a/docs/guides/code_examples/ai_crawler/additional_instructions_example.py b/docs/guides/code_examples/ai_crawler/additional_instructions_example.py new file mode 100644 index 0000000000..aae0397da9 --- /dev/null +++ b/docs/guides/code_examples/ai_crawler/additional_instructions_example.py @@ -0,0 +1,44 @@ +import asyncio + +from pydantic import BaseModel +from pydantic_ai.models.openai import OpenAIChatModel +from pydantic_ai.providers.openai import OpenAIProvider + +from crawlee.crawlers import AiCrawler, AiCrawlingContext + + +class Post(BaseModel): + """Model representing a single post.""" + + title: str + url: str + + +class Posts(BaseModel): + """Model representing the extracted list of posts.""" + + posts: list[Post] + + +async def main() -> None: + model = OpenAIChatModel( + 'gpt-5.4-nano', + provider=OpenAIProvider(api_key='your-openai-api-key'), + ) + crawler = AiCrawler(model=model, max_requests_per_crawl=5) + + @crawler.router.default_handler + async def handler(context: AiCrawlingContext) -> None: + # The instruction narrows what the model returns from the page. + posts = await context.extract( + Posts, + additional_instructions='Extract only the top five posts on the page.', + ) + + await context.push_data(posts.model_dump()) + + await crawler.run(['https://news.ycombinator.com']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/ai_crawler/basic_example.py b/docs/guides/code_examples/ai_crawler/basic_example.py new file mode 100644 index 0000000000..7cdd458ce7 --- /dev/null +++ b/docs/guides/code_examples/ai_crawler/basic_example.py @@ -0,0 +1,41 @@ +import asyncio + +from pydantic import BaseModel +from pydantic_ai.models.openai import OpenAIChatModel +from pydantic_ai.providers.openai import OpenAIProvider + +from crawlee.crawlers import AiCrawler, AiCrawlingContext + + +class Article(BaseModel): + """Model representing the extracted data for an article.""" + + title: str + short_text: str + + +async def main() -> None: + model = OpenAIChatModel( + 'gpt-5.4-nano', + # Set the provider with the API key explicitly. + provider=OpenAIProvider(api_key='your-openai-api-key'), + ) + + crawler = AiCrawler(model=model, max_requests_per_crawl=5) + + @crawler.router.default_handler + async def handler(context: AiCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url} ...') + + # Pass a Pydantic model and get a validated instance back. + article = await context.extract(Article) + + await context.push_data(article.model_dump()) + + await context.enqueue_links() + + await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/ai_crawler/custom_distiller_example.py b/docs/guides/code_examples/ai_crawler/custom_distiller_example.py new file mode 100644 index 0000000000..fb1faca290 --- /dev/null +++ b/docs/guides/code_examples/ai_crawler/custom_distiller_example.py @@ -0,0 +1,67 @@ +import asyncio + +from html_to_markdown import convert +from lxml_html_clean import Cleaner +from pydantic import BaseModel +from pydantic_ai.models.openai import OpenAIChatModel +from pydantic_ai.providers.openai import OpenAIProvider + +from crawlee.crawlers import ( + AiCrawler, + AiCrawlingContext, + AiDirectExtractor, + BaseAiHtmlDistiller, + get_basic_ai_cleaner, +) + +# Notes appended to the model instructions so it knows the input format. +MARKDOWN_PROMPT_NOTES = 'The document is Markdown converted from the HTML page.' + + +class MarkdownDistiller(BaseAiHtmlDistiller): + """Distiller that cleans the page HTML and converts it to Markdown.""" + + def __init__(self, cleaner: Cleaner | None = None) -> None: + super().__init__(prompt_notes=MARKDOWN_PROMPT_NOTES) + + # Strip scripts, styles, and other noise before the conversion. + self._cleaner = cleaner or get_basic_ai_cleaner() + + def distill(self, html: str) -> str: + return convert(self._cleaner.clean_html(html)).content or '' + + +class Article(BaseModel): + """Model representing the extracted data for an article.""" + + title: str + short_text: str + + +async def main() -> None: + model = OpenAIChatModel( + 'gpt-5.4-nano', + # Set the provider with the API key explicitly. + provider=OpenAIProvider(api_key='your-openai-api-key'), + ) + crawler = AiCrawler( + # Use the custom distiller to convert the page to Markdown before extraction. + extractor=AiDirectExtractor(model=model, distiller=MarkdownDistiller()), + max_requests_per_crawl=5, + ) + + @crawler.router.default_handler + async def handler(context: AiCrawlingContext) -> None: + # Pass a Pydantic model and get a validated instance back. + article = await context.extract(Article) + await context.push_data(article.model_dump()) + + # Enqueue links as usual, the distillation and extraction don't affect + # the rest of the crawling logic. + await context.enqueue_links() + + await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/ai_crawler/selector_extractor_example.py b/docs/guides/code_examples/ai_crawler/selector_extractor_example.py new file mode 100644 index 0000000000..f6dd98eb39 --- /dev/null +++ b/docs/guides/code_examples/ai_crawler/selector_extractor_example.py @@ -0,0 +1,56 @@ +import asyncio + +from pydantic import BaseModel +from pydantic_ai.models.openai import OpenAIChatModel +from pydantic_ai.providers.openai import OpenAIProvider + +from crawlee import Glob +from crawlee.crawlers import ( + AiCrawler, + AiCrawlingContext, + AiDirectExtractor, + AiSelectorExtractor, +) + + +class Article(BaseModel): + """Model representing the extracted data for an article.""" + + title: str + main_text: str + + +async def main() -> None: + model = OpenAIChatModel( + 'gpt-5.4-nano', + provider=OpenAIProvider(api_key='your-openai-api-key'), + ) + crawler = AiCrawler( + extractor=AiSelectorExtractor( + model=model, + # Pages the cached selectors cannot handle fall back to direct extraction. + fallback=AiDirectExtractor(model=model), + ), + max_requests_per_crawl=10, + ) + + @crawler.router.default_handler + async def handler(context: AiCrawlingContext) -> None: + # Enqueue blog article pages; the article handler extracts the data. + await context.enqueue_links( + include=[Glob('https://crawlee.dev/blog/*')], + label='article', + ) + + @crawler.router.handler('article') + async def article_handler(context: AiCrawlingContext) -> None: + # The first page generates selectors; later pages reuse them with no LLM call. + article = await context.extract(Article) + + await context.push_data(article.model_dump()) + + await crawler.run(['https://crawlee.dev/blog']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/ai_crawler/usage_limit_example.py b/docs/guides/code_examples/ai_crawler/usage_limit_example.py new file mode 100644 index 0000000000..7b0985af2f --- /dev/null +++ b/docs/guides/code_examples/ai_crawler/usage_limit_example.py @@ -0,0 +1,57 @@ +import asyncio + +from pydantic import BaseModel +from pydantic_ai.exceptions import UsageLimitExceeded +from pydantic_ai.models.openai import OpenAIChatModel +from pydantic_ai.providers.openai import OpenAIProvider +from pydantic_ai.usage import UsageLimits + +from crawlee.crawlers import AiCrawler, AiCrawlingContext, AiDirectExtractor + +# Stop the whole crawl once this many tokens have been spent. +TOKEN_BUDGET = 50_000 + + +class Article(BaseModel): + """Model representing the extracted data for an article.""" + + title: str + short_text: str + + +async def main() -> None: + model = OpenAIChatModel( + 'gpt-5.4-nano', + provider=OpenAIProvider(api_key='your-openai-api-key'), + ) + crawler = AiCrawler( + # Cap each extraction so an oversized page cannot consume LLM resources. + extractor=AiDirectExtractor( + model=model, + usage_limits=UsageLimits(total_tokens_limit=10_000), + ), + max_requests_per_crawl=5, + ) + + @crawler.router.default_handler + async def handler(context: AiCrawlingContext) -> None: + # Stop the crawl once the cumulative token budget is exhausted. + if context.ai_usage.total_tokens > TOKEN_BUDGET: + context.log.info('Token budget exhausted, stopping the crawler.') + crawler.stop() + return + + try: + article = await context.extract(Article) + except UsageLimitExceeded: + # The page needs more tokens than the per-extraction limit allows. + context.log.warning(f'Content at {context.request.url} is too large.') + return + + await context.push_data(article.model_dump()) + + await crawler.run(['https://crawlee.dev/']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/src/crawlee/crawlers/_ai/_ai_crawler.py b/src/crawlee/crawlers/_ai/_ai_crawler.py index 5c89d20e1f..c4a92ea56e 100644 --- a/src/crawlee/crawlers/_ai/_ai_crawler.py +++ b/src/crawlee/crawlers/_ai/_ai_crawler.py @@ -29,6 +29,9 @@ logger = getLogger(__name__) +# Default model +_DEFAULT_AI_MODEL = 'openai:gpt-5.4-nano' + @docs_group('Crawlers') class AiCrawler(AbstractHttpCrawler[AiCrawlingContext, Selector, Selector]): @@ -86,20 +89,18 @@ def __init__( model: The model used for extraction, given to the default extractor (`AiDirectExtractor`). A provider-prefixed name (e.g. `'openai:gpt-5.4-nano'`) or a Pydantic AI `Model` instance. When given as a string, the provider reads credentials from its environment variable (e.g. `OPENAI_API_KEY`). - Pass a `Model` instance to supply them explicitly. Provide exactly one of `model` or `extractor`. + Pass a `Model` instance to supply them explicitly. Defaults to `'openai:gpt-5.4-nano'` when neither + `model` nor `extractor` is given. Provide at most one of `model` or `extractor`. extractor: A pre-configured `AiHtmlExtractor`, for full control over the distiller, instructions, caching, usage limits, and model fallback. Pass an `AiSelectorExtractor` here for cached-selector - extraction. Provide exactly one of `model` or `extractor`. + extraction. Provide at most one of `model` or `extractor`. kwargs: Additional keyword arguments to pass to the underlying `AbstractHttpCrawler`. """ - if (model is None) == (extractor is None): - raise ValueError('Provide exactly one of `model` or `extractor`.') - - if extractor is None and model is not None: - extractor = AiDirectExtractor(model) + if model is not None and extractor is not None: + raise ValueError('Provide at most one of `model` or `extractor`.') - if not extractor: - raise ValueError('Extractor initialization failed; check the provided model or extractor configuration.') + if extractor is None: + extractor = AiDirectExtractor(model if model is not None else _DEFAULT_AI_MODEL) # Call the notification only once. warnings.warn( diff --git a/tests/unit/crawlers/_ai/test_ai_crawler.py b/tests/unit/crawlers/_ai/test_ai_crawler.py index 624e0c5e27..ada696edc0 100644 --- a/tests/unit/crawlers/_ai/test_ai_crawler.py +++ b/tests/unit/crawlers/_ai/test_ai_crawler.py @@ -23,11 +23,8 @@ class _Article(BaseModel): title: str -def test_requires_exactly_one_of_model_or_extractor() -> None: - with pytest.raises(ValueError, match='exactly one'): - AiCrawler() - - with pytest.raises(ValueError, match='exactly one'): +def test_rejects_model_and_extractor_together() -> None: + with pytest.raises(ValueError, match='at most one'): AiCrawler(model=TestModel(), extractor=AiDirectExtractor(TestModel())) @@ -35,6 +32,14 @@ def test_default_extractor_is_direct() -> None: assert isinstance(AiCrawler(model=TestModel()).extractor, AiDirectExtractor) +def test_default_model_when_none_given(monkeypatch: pytest.MonkeyPatch) -> None: + # The default model is an OpenAI one, whose client needs a key at construction. + monkeypatch.setenv('OPENAI_API_KEY', 'test-key') + + # With neither model nor extractor, the crawler builds the default direct extractor. + assert isinstance(AiCrawler().extractor, AiDirectExtractor) + + def test_emits_experimental_warning() -> None: with pytest.warns(UserWarning, match='experimental'): AiCrawler(model=TestModel())