feat(pry): production readiness pass — apify actor, async db, retry wiring, tests, observability, mypy
- Pin Dockerfile --workers 1 - Wire retry.py + circuit breakers into ultimate_scraper tiers - Add Apify actor (apify_actor.py, Dockerfile.apify, .actor/actor.json) - Add async SQLAlchemy support alongside sync db.py - Add 31 HTTP integration tests (tests/test_api_integration.py + test_api_mcp.py) - Add OTLP exporter support in observability.py - Re-enable mypy var-annotated error code; fix annotations - Improve CI workflow (pip cache, install -e .[dev], gitleaks, commitlint, 40% coverage gate)
This commit is contained in:
parent
5a21133b1d
commit
345cd79bc9
21 changed files with 1979 additions and 44 deletions
|
|
@ -506,7 +506,9 @@ class PryScraper:
|
|||
opts = options or {}
|
||||
max_pages = opts.get("max_pages", 10)
|
||||
max_depth = opts.get("max_depth", 2)
|
||||
visited, to_visit, results = set(), [(url, 0)], []
|
||||
visited: set[str] = set()
|
||||
to_visit: list[tuple[str, int]] = [(url, 0)]
|
||||
results: list[Any] = []
|
||||
|
||||
while to_visit and len(visited) < max_pages:
|
||||
current_url, depth = to_visit.pop(0)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue