diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index be3da6dd27..a77ae77294 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -26,7 +26,7 @@ from crawlee import Glob, RequestTransformAction, service_locator from crawlee._autoscaling import AutoscaledPool, Snapshotter, SystemStatus -from crawlee._log_config import configure_logger, get_configured_log_level, string_to_log_level +from crawlee._log_config import configure_logger, string_to_log_level from crawlee._request import Request, RequestOptions, RequestState from crawlee._service_locator import ServiceLocator from crawlee._types import ( @@ -179,7 +179,8 @@ class _BasicCrawlerOptions(TypedDict): """If True, the crawler stops immediately when any request handler error occurs.""" configure_logging: NotRequired[bool] - """If True, the crawler will set up logging infrastructure automatically.""" + """If True, the crawler will set up logging infrastructure automatically, unless the root logger already has + handlers configured by the host application.""" statistics_log_format: NotRequired[Literal['table', 'inline']] """If 'table', displays crawler statistics as formatted tables in logs. If 'inline', outputs statistics as plain @@ -349,7 +350,8 @@ def __init__( abort_on_error: If True, the crawler stops immediately when any request handler error occurs. keep_alive: If True, it will keep crawler alive even if there are no requests in queue. Use `crawler.stop()` to exit the crawler. - configure_logging: If True, the crawler will set up logging infrastructure automatically. + configure_logging: If True, the crawler will set up logging infrastructure automatically, unless the root + logger already has handlers configured by the host application. statistics_log_format: If 'table', displays crawler statistics as formatted tables in logs. If 'inline', outputs statistics as plain text log messages. respect_robots_txt_file: If set to `True`, the crawler will automatically try to fetch the robots.txt file @@ -451,9 +453,10 @@ def __init__( # Logging setup if configure_logging: root_logger = logging.getLogger() - configure_logger(root_logger, remove_old_handlers=True) - httpx_logger = logging.getLogger('httpx') # Silence HTTPX logger - httpx_logger.setLevel(logging.DEBUG if get_configured_log_level() <= logging.DEBUG else logging.WARNING) + # Leave the loggers untouched if the host application has already configured the root logger, + # mirroring `logging.basicConfig` semantics. + if not root_logger.handlers: + configure_logger(root_logger) self._logger = _logger or logging.getLogger(__name__) if implicit_event_manager_with_explicit_config: self._logger.warning( diff --git a/src/crawlee/http_clients/_httpx.py b/src/crawlee/http_clients/_httpx.py index f2c2565125..2b4c6f2e52 100644 --- a/src/crawlee/http_clients/_httpx.py +++ b/src/crawlee/http_clients/_httpx.py @@ -2,12 +2,13 @@ import asyncio from contextlib import asynccontextmanager -from logging import getLogger +from logging import DEBUG, WARNING, getLogger from typing import TYPE_CHECKING, Any, cast import httpx from typing_extensions import override +from crawlee._log_config import get_configured_log_level from crawlee._types import HttpHeaders from crawlee._utils.blocked import ROTATE_PROXY_ERRORS from crawlee._utils.docs import docs_group @@ -128,6 +129,12 @@ def __init__( super().__init__( persist_cookies_per_session=persist_cookies_per_session, ) + + # `httpx` logs one INFO line per request, which is too noisy for the default log level. Silence it down to + # WARNING unless the user has explicitly opted into DEBUG. + httpx_logger = getLogger('httpx') + httpx_logger.setLevel(DEBUG if get_configured_log_level() <= DEBUG else WARNING) + self._http1 = http1 self._http2 = http2 diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 0391d65843..526a24b980 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -20,6 +20,7 @@ import pytest from crawlee import ConcurrencySettings, Glob, service_locator +from crawlee._log_config import CrawleeLogFormatter from crawlee._request import Request, RequestState from crawlee._types import BasicCrawlingContext, EnqueueLinksKwargs, HttpMethod from crawlee._utils.robots import RobotsTxtFile @@ -1053,6 +1054,40 @@ def test_crawler_log() -> None: crawler.log.info('Test log message') +def test_configure_logging_preserves_host_app_root_handlers() -> None: + """Constructing a crawler must not remove handlers installed on the root logger by the host application.""" + root_logger = logging.getLogger() + host_handler = logging.NullHandler() + root_logger.addHandler(host_handler) + + try: + BasicCrawler() + assert host_handler in root_logger.handlers + # The host-configured root logger must not gain Crawlee's own handler either. + assert not any(isinstance(handler.formatter, CrawleeLogFormatter) for handler in root_logger.handlers) + finally: + root_logger.removeHandler(host_handler) + + +def test_configure_logging_sets_up_unconfigured_root_logger() -> None: + """Constructing a crawler installs Crawlee's log handler when the root logger has no handlers yet.""" + root_logger = logging.getLogger() + saved_handlers = root_logger.handlers[:] + saved_level = root_logger.level + for handler in saved_handlers: + root_logger.removeHandler(handler) + + try: + BasicCrawler() + assert any(isinstance(handler.formatter, CrawleeLogFormatter) for handler in root_logger.handlers) + finally: + for handler in root_logger.handlers[:]: + root_logger.removeHandler(handler) + for handler in saved_handlers: + root_logger.addHandler(handler) + root_logger.setLevel(saved_level) + + async def test_consecutive_runs_purge_request_queue() -> None: crawler = BasicCrawler() visit = Mock() diff --git a/tests/unit/http_clients/test_httpx.py b/tests/unit/http_clients/test_httpx.py index a200d28f77..c98ca4bbf7 100644 --- a/tests/unit/http_clients/test_httpx.py +++ b/tests/unit/http_clients/test_httpx.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import logging from typing import TYPE_CHECKING import pytest @@ -23,6 +24,16 @@ async def http_client() -> AsyncGenerator[HttpClient]: yield client +def test_silences_httpx_request_logging() -> None: + """Instantiating the client lowers the noisy per-request `httpx` INFO logs to WARNING by default.""" + httpx_logger = logging.getLogger('httpx') + httpx_logger.setLevel(logging.NOTSET) + + HttpxHttpClient() + + assert httpx_logger.level == logging.WARNING + + async def test_common_headers_and_user_agent(server_url: URL, header_network: dict) -> None: """Test that the relevant headers use header values from header generator instead of default Httpx headers.