diff --git a/docs/03_guides/14_ai_agents.mdx b/docs/03_guides/14_ai_agents.mdx index f235e3ff..d1733357 100644 --- a/docs/03_guides/14_ai_agents.mdx +++ b/docs/03_guides/14_ai_agents.mdx @@ -156,6 +156,7 @@ Note that: - `OpenAILike` points `api_base` at the Apify OpenRouter proxy, so no provider API key is needed. It's the LlamaIndex LLM class for any OpenAI-compatible endpoint. - `Document` wraps each scraped page, and `VectorStoreIndex.from_documents` chunks and embeds them. The index is the retrieval layer that sets LlamaIndex apart from a plain agent loop. - Embeddings run locally through `HuggingFaceEmbedding` from the `llama-index-embeddings-huggingface` package, so the proxy needs no embeddings endpoint. +- The embedding model downloads on the first run and the Website Content Crawler renders pages in a browser, so this pipeline is heavier than the other examples. - `index.as_query_engine(output_cls=Answer)` retrieves the passages most relevant to the question, then the LLM composes a validated `Answer` grounded in them. - Every response carries the passages it used in `response.source_nodes`, so the Actor pushes their `url` metadata as citations. `Actor.call` runs the Website Content Crawler, and `to_document` maps each dataset item to a `Document`, so indexing more content is a matter of crawling more pages. For details, see [Using Apify Actors as tools](#using-apify-actors-as-tools). diff --git a/docs/03_guides/code/14_crewai.py b/docs/03_guides/code/14_crewai.py index d1b7875a..38b62169 100644 --- a/docs/03_guides/code/14_crewai.py +++ b/docs/03_guides/code/14_crewai.py @@ -77,8 +77,15 @@ async def main() -> None: # to fetch the page as clean Markdown. researcher = Agent( role='Documentation researcher', - goal='Read the Crawlee docs and note every crawler it describes.', - backstory='A researcher who reads technical docs closely.', + goal=( + 'Fetch the page with the web_browser tool and note every crawler ' + 'it describes.' + ), + backstory=( + 'A researcher who never answers from memory. Before writing ' + 'anything, always read the actual page with the web_browser tool ' + 'and report only what it says.' + ), tools=[WebBrowserTool()], llm=llm, ) @@ -91,7 +98,10 @@ async def main() -> None: ) research = Task( - description=f'Scrape {url} and list the crawlers the page covers.', + description=( + f'Use the web_browser tool to fetch {url}. Based only on the returned ' + 'Markdown, list the crawlers the page covers. Do not use prior knowledge.' + ), expected_output='Notes on each crawler: name, what it builds on, its use.', agent=researcher, ) diff --git a/website/roa-loader/index.js b/website/roa-loader/index.js index d3a3fe68..1a0af6d8 100644 --- a/website/roa-loader/index.js +++ b/website/roa-loader/index.js @@ -29,7 +29,7 @@ async function getHash(source) { build: 'latest', contentType: 'application/json; charset=utf-8', memory, - timeout: 180, + timeout: 360, }, }), headers: { diff --git a/website/versioned_docs/version-3.4/03_guides/14_ai_agents.mdx b/website/versioned_docs/version-3.4/03_guides/14_ai_agents.mdx index f235e3ff..d1733357 100644 --- a/website/versioned_docs/version-3.4/03_guides/14_ai_agents.mdx +++ b/website/versioned_docs/version-3.4/03_guides/14_ai_agents.mdx @@ -156,6 +156,7 @@ Note that: - `OpenAILike` points `api_base` at the Apify OpenRouter proxy, so no provider API key is needed. It's the LlamaIndex LLM class for any OpenAI-compatible endpoint. - `Document` wraps each scraped page, and `VectorStoreIndex.from_documents` chunks and embeds them. The index is the retrieval layer that sets LlamaIndex apart from a plain agent loop. - Embeddings run locally through `HuggingFaceEmbedding` from the `llama-index-embeddings-huggingface` package, so the proxy needs no embeddings endpoint. +- The embedding model downloads on the first run and the Website Content Crawler renders pages in a browser, so this pipeline is heavier than the other examples. - `index.as_query_engine(output_cls=Answer)` retrieves the passages most relevant to the question, then the LLM composes a validated `Answer` grounded in them. - Every response carries the passages it used in `response.source_nodes`, so the Actor pushes their `url` metadata as citations. `Actor.call` runs the Website Content Crawler, and `to_document` maps each dataset item to a `Document`, so indexing more content is a matter of crawling more pages. For details, see [Using Apify Actors as tools](#using-apify-actors-as-tools). diff --git a/website/versioned_docs/version-3.4/03_guides/code/14_crewai.py b/website/versioned_docs/version-3.4/03_guides/code/14_crewai.py index d1b7875a..38b62169 100644 --- a/website/versioned_docs/version-3.4/03_guides/code/14_crewai.py +++ b/website/versioned_docs/version-3.4/03_guides/code/14_crewai.py @@ -77,8 +77,15 @@ async def main() -> None: # to fetch the page as clean Markdown. researcher = Agent( role='Documentation researcher', - goal='Read the Crawlee docs and note every crawler it describes.', - backstory='A researcher who reads technical docs closely.', + goal=( + 'Fetch the page with the web_browser tool and note every crawler ' + 'it describes.' + ), + backstory=( + 'A researcher who never answers from memory. Before writing ' + 'anything, always read the actual page with the web_browser tool ' + 'and report only what it says.' + ), tools=[WebBrowserTool()], llm=llm, ) @@ -91,7 +98,10 @@ async def main() -> None: ) research = Task( - description=f'Scrape {url} and list the crawlers the page covers.', + description=( + f'Use the web_browser tool to fetch {url}. Based only on the returned ' + 'Markdown, list the crawlers the page covers. Do not use prior knowledge.' + ), expected_output='Notes on each crawler: name, what it builds on, its use.', agent=researcher, )