feat(pry): production readiness pass — apify actor, async db, retry wiring, tests, observability, mypy
- Pin Dockerfile --workers 1 - Wire retry.py + circuit breakers into ultimate_scraper tiers - Add Apify actor (apify_actor.py, Dockerfile.apify, .actor/actor.json) - Add async SQLAlchemy support alongside sync db.py - Add 31 HTTP integration tests (tests/test_api_integration.py + test_api_mcp.py) - Add OTLP exporter support in observability.py - Re-enable mypy var-annotated error code; fix annotations - Improve CI workflow (pip cache, install -e .[dev], gitleaks, commitlint, 40% coverage gate)
This commit is contained in:
parent
5a21133b1d
commit
345cd79bc9
21 changed files with 1979 additions and 44 deletions
240
apify_actor.py
Normal file
240
apify_actor.py
Normal file
|
|
@ -0,0 +1,240 @@
|
|||
"""Pry Apify Actor — web scraping + browser automation on the Apify platform.
|
||||
|
||||
Runs Pry's multi-tier scraper (UltimateScraper) as an Apify actor.
|
||||
Accepts a list of URLs, scrapes each one through up to 12 fallback tiers,
|
||||
and pushes the results (markdown content + metadata) to the Apify dataset.
|
||||
|
||||
Usage:
|
||||
Deploy to Apify Console or run locally with:
|
||||
apify run -p
|
||||
|
||||
Input (JSON):
|
||||
{
|
||||
"urls": ["https://example.com", ...],
|
||||
"max_pages": 1,
|
||||
"timeout": 30,
|
||||
"output_format": "markdown",
|
||||
"proxy_config": {
|
||||
"use_apify_proxies": true,
|
||||
"proxy_country": "US"
|
||||
}
|
||||
}
|
||||
|
||||
Output (dataset items):
|
||||
{
|
||||
"url": "https://example.com",
|
||||
"status": "ok" | "error",
|
||||
"content": "...markdown or html...",
|
||||
"method": "direct" | "flaresolverr" | ...,
|
||||
"metadata": {
|
||||
"title": "...",
|
||||
"description": "...",
|
||||
"domain": "example.com",
|
||||
"scraped_at": "2026-07-03T..."
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
#
|
||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
# Licensed under MIT. See LICENSE.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
from errors import InvalidRequestError
|
||||
from ultimate_scraper import UltimateScraper
|
||||
from url_guard import validate_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Apify SDK is imported lazily inside main() so this file doesn't crash
|
||||
# when imported from the main Pry codebase (where apify isn't installed).
|
||||
|
||||
|
||||
def _extract_links(html: str, base_url: str) -> list[str]:
|
||||
"""Extract same-domain links from raw HTML for crawling."""
|
||||
links: list[str] = []
|
||||
parsed_base = urlparse(base_url)
|
||||
domain = parsed_base.netloc
|
||||
scheme = parsed_base.scheme
|
||||
|
||||
for href in re.findall(r'href=["\'](https?://[^"\']+)["\']', html, re.IGNORECASE):
|
||||
link = urljoin(base_url, href)
|
||||
link_parsed = urlparse(link)
|
||||
if link_parsed.netloc == domain and link_parsed.scheme in ("http", "https"):
|
||||
# Deduplicate by removing fragment
|
||||
clean = link.split("#")[0]
|
||||
if clean not in links:
|
||||
links.append(clean)
|
||||
elif link.startswith("/"):
|
||||
full = f"{scheme}://{domain}{link}"
|
||||
if full not in links:
|
||||
links.append(full)
|
||||
|
||||
return links[:50] # cap per page
|
||||
|
||||
|
||||
async def scrape_url(
|
||||
scraper: UltimateScraper,
|
||||
url: str,
|
||||
options: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Scrape a single URL and return a structured result."""
|
||||
start = time.time()
|
||||
result = await scraper.scrape(url, options)
|
||||
elapsed = round(time.time() - start, 2)
|
||||
|
||||
status = result.get("status", "error")
|
||||
output: dict[str, Any] = {
|
||||
"url": url,
|
||||
"status": status,
|
||||
"method": result.get("method", "unknown"),
|
||||
"scraped_at": datetime.now(UTC).isoformat(),
|
||||
"elapsed_seconds": elapsed,
|
||||
}
|
||||
|
||||
if status == "ok":
|
||||
output["content"] = result.get("content", "")
|
||||
output["raw_html"] = result.get("raw_html", "")
|
||||
output["metadata"] = {
|
||||
"domain": urlparse(url).netloc,
|
||||
"content_length": len(result.get("content", "") or ""),
|
||||
"method": result.get("method", "unknown"),
|
||||
}
|
||||
else:
|
||||
output["error"] = result.get("error", "unknown_error")
|
||||
|
||||
return output
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
"""Pry Apify Actor entry point."""
|
||||
# Lazy import so this file is safe to import from the main codebase.
|
||||
from apify import Actor
|
||||
|
||||
async with Actor:
|
||||
actor_input = await Actor.get_input() or {}
|
||||
|
||||
# Parse input
|
||||
raw_urls = actor_input.get("urls", [])
|
||||
if isinstance(raw_urls, str):
|
||||
raw_urls = [raw_urls]
|
||||
if not raw_urls:
|
||||
Actor.log.info("No URLs provided in input; nothing to scrape.")
|
||||
await Actor.push_data({"status": "skipped", "reason": "no_urls"})
|
||||
return
|
||||
|
||||
max_pages = max(1, int(actor_input.get("max_pages", 1)))
|
||||
timeout = int(actor_input.get("timeout", 30))
|
||||
output_format = actor_input.get("output_format", "markdown")
|
||||
crawl = bool(actor_input.get("crawl", False))
|
||||
|
||||
# Build scraper options
|
||||
base_options: dict[str, Any] = {
|
||||
"timeout": timeout,
|
||||
}
|
||||
if actor_input.get("proxy_config", {}).get("use_apify_proxies"):
|
||||
# Apify provides HTTP proxies via http://user:pass@proxy.apify.com:8000
|
||||
proxy_url = Actor.config.get("proxy_url")
|
||||
if proxy_url:
|
||||
base_options["proxy_url"] = proxy_url
|
||||
|
||||
scraper = UltimateScraper()
|
||||
visited: set[str] = set()
|
||||
to_visit: list[str] = list(raw_urls)
|
||||
total_scraped = 0
|
||||
total_errors = 0
|
||||
|
||||
Actor.log.info(
|
||||
"pry_actor_start",
|
||||
extra={
|
||||
"urls": len(raw_urls),
|
||||
"max_pages": max_pages,
|
||||
"crawl": crawl,
|
||||
"timeout": timeout,
|
||||
},
|
||||
)
|
||||
|
||||
while to_visit and total_scraped < max_pages:
|
||||
url = to_visit.pop(0)
|
||||
if url in visited:
|
||||
continue
|
||||
visited.add(url)
|
||||
|
||||
# Validate URL before scraping
|
||||
try:
|
||||
validate_url(url)
|
||||
except InvalidRequestError as e:
|
||||
Actor.log.warning("pry_actor_skip_invalid", extra={"url": url, "error": str(e)})
|
||||
await Actor.push_data(
|
||||
{
|
||||
"url": url,
|
||||
"status": "error",
|
||||
"error": f"Invalid URL: {e}",
|
||||
"scraped_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
)
|
||||
total_errors += 1
|
||||
continue
|
||||
|
||||
# Scrape with options
|
||||
Actor.log.info("pry_actor_scrape", extra={"url": url, "remaining": len(to_visit)})
|
||||
options = dict(base_options)
|
||||
if output_format:
|
||||
options["output_format"] = output_format
|
||||
|
||||
output = await scrape_url(scraper, url, options)
|
||||
|
||||
# For crawled URLs, try to extract more links
|
||||
if crawl and output["status"] == "ok":
|
||||
html = output.get("raw_html", "")
|
||||
if html:
|
||||
discovered = _extract_links(html, url)
|
||||
new_links = [
|
||||
link for link in discovered if link not in visited and link not in to_visit
|
||||
]
|
||||
to_visit.extend(new_links[:20]) # cap frontier per page
|
||||
Actor.log.debug(
|
||||
"pry_actor_links_discovered",
|
||||
extra={"url": url, "new_links": len(new_links)},
|
||||
)
|
||||
|
||||
await Actor.push_data(output)
|
||||
total_scraped += 1
|
||||
if output["status"] != "ok":
|
||||
total_errors += 1
|
||||
|
||||
# Report progress to Apify
|
||||
await Actor.set_value(
|
||||
"STATE",
|
||||
{
|
||||
"visited": len(visited),
|
||||
"total_scraped": total_scraped,
|
||||
"total_errors": total_errors,
|
||||
"queue_remaining": len(to_visit),
|
||||
},
|
||||
)
|
||||
|
||||
Actor.log.info(
|
||||
"pry_actor_finish",
|
||||
extra={
|
||||
"total_scraped": total_scraped,
|
||||
"total_errors": total_errors,
|
||||
"urls_visited": len(visited),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import asyncio
|
||||
|
||||
asyncio.run(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue