RugMunchMedia/pryscraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
pryscraper/templates/test_live.py
cryptorugmunch 0200bf3e16 refactor(exceptions): add ruff BLE001; convert 103 broad except Exception 
Per CONVENTIONS.md Part 2 ("Never bare except") and CONVENTIONS.md
Part 7 (pre-commit hooks: ruff), blind `except Exception` is now a
lint failure. Pre-existing sites are marked `# noqa: BLE001` for
later manual review; new code must use specific exception types.

Changes:
- pyproject.toml: added "BLE" to ruff lint select. BLE001 is now enforced
- 103 of 166 `except Exception` sites were auto-converted to specific
  types based on context (httpx, json, OSError, subprocess, etc.)
- 62 remaining sites marked with `# noqa: BLE001` for later review
  (mostly generic try/except wrappers that legitimately need broad catch
  for graceful degradation: e.g. compliance LLM fallback must catch
  any error to preserve the regex result)
- 1 manual fix: reverted compliance.py LLM fallback to broad except
  with explicit "must catch all errors" comment + noqa
- 2 files (commerce_sync.py, crm_sync.py) needed `import httpx` added
  so the auto-converted exception references would resolve
- 5 source files (agency, monitor, pipelines, auth_connector,
  llm_providers/registry) renamed "name" -> "<scope>_name" in
  extra={...} dicts because "name" is a reserved LogRecord field

Test impact:
- 14 failing tests -> 1 (the SSE subprocess test is a sandbox limitation,
  pre-existing and unrelated)
- New `test_ble_temp.py` verifies BLE001 catches new violations

Follow-up:
- Each `# noqa: BLE001` site should be reviewed and replaced with a
  specific exception type where possible. The most common legitimate
  broad-catch case is the LLM fallback path; everything else probably
  can be narrowed.
2026-07-02 21:04:53 +02:00

156 lines
6.9 KiB
Python
Raw Blame History

#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 Rug Munch Media LLC
#
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
# Licensed under MIT. See LICENSE.
"""Live template verification script — run on Talos with network access."""
import asyncio
import sys
import httpx
TEMPLATES = [
("abcnews-article", "abcnews.go.com", "news"),
("adidas-product", "adidas.com", "ecommerce"),
("airbnb-listing", "airbnb.com", "travel"),
("aliexpress-product", "aliexpress.com", "ecommerce"),
("aljazeera-article", "aljazeera.com", "news"),
("amazon-product", "amazon.com", "ecommerce"),
("amazon-search", "amazon.com", "ecommerce"),
("arstechnica-article", "arstechnica.com", "news"),
("asos-product", "asos.com", "ecommerce"),
("associated-press-article", "apnews.com", "news"),
("axios-article", "axios.com", "news"),
("bbc-article", "bbc.com", "news"),
("bestbuy-product", "bestbuy.com", "ecommerce"),
("bhphotovideo-product", "bhphotovideo.com", "ecommerce"),
("bloomberg-article", "bloomberg.com", "news"),
("booking-hotel", "booking.com", "travel"),
("businessinsider-article", "businessinsider.com", "news"),
("cbsnews-article", "cbsnews.com", "news"),
("chewy-product", "chewy.com", "ecommerce"),
("cnet-article", "cnet.com", "news"),
("cnn-article", "cnn.com", "news"),
("coinmarketcap-coin", "coinmarketcap.com", "financial"),
("costco-product", "costco.com", "ecommerce"),
("craigslist-listing", "craigslist.org", "classifieds"),
("crate-and-barrel-product", "crateandbarrel.com", "ecommerce"),
("crunchbase-company", "crunchbase.com", "business"),
("cvs-product", "cvs.com", "ecommerce"),
("dailymail-article", "dailymail.co.uk", "news"),
("dicks-sporting-goods", "dickssportinggoods.com", "ecommerce"),
("dw-article", "dw.com", "news"),
("ebay-listing", "ebay.com", "ecommerce"),
("economist-article", "economist.com", "news"),
("espn-sports", "espn.com", "sports"),
("etsy-product", "etsy.com", "ecommerce"),
("facebook-marketplace", "facebook.com", "classifieds"),
("financialtimes-article", "ft.com", "news"),
("forbes-article", "forbes.com", "news"),
("foxnews-article", "foxnews.com", "news"),
("france24-article", "france24.com", "news"),
("g2-product", "g2.com", "reviews"),
("gap-product", "gap.com", "ecommerce"),
("generic-product", "generic", "ecommerce"),
("github-repo", "github.com", "development"),
("glassdoor-job", "glassdoor.com", "professional"),
("google-maps-place", "google.com/maps", "local"),
("google-patent", "patents.google.com", "reference"),
("h-and-m-product", "hm.com", "ecommerce"),
("hackernews-post", "news.ycombinator.com", "news"),
("homedepot-product", "homedepot.com", "ecommerce"),
("ikea-product", "ikea.com", "ecommerce"),
("imdb-movie", "imdb.com", "media"),
("indeed-job", "indeed.com", "professional"),
("jcpenney-product", "jcpenney.com", "ecommerce"),
("kohls-product", "kohls.com", "ecommerce"),
("linkedin-job", "linkedin.com", "professional"),
("linkedin-profile", "linkedin.com", "professional"),
("lowes-product", "lowes.com", "ecommerce"),
("lululemon-product", "lululemon.com", "ecommerce"),
("macys-product", "macys.com", "ecommerce"),
("medium-article", "medium.com", "content"),
("monster-job", "monster.com", "professional"),
("nbcnews-article", "nbcnews.com", "news"),
("newegg-product", "newegg.com", "ecommerce"),
("nike-product", "nike.com", "ecommerce"),
("nordstrom-product", "nordstrom.com", "ecommerce"),
("nytimes-article", "nytimes.com", "news"),
("officedepot-product", "officedepot.com", "ecommerce"),
("old-navy-product", "oldnavy.gap.com", "ecommerce"),
("petsmart-product", "petsmart.com", "ecommerce"),
("politico-article", "politico.com", "news"),
("pottery-barn-product", "potterybarn.com", "ecommerce"),
("producthunt-product", "producthunt.com", "business"),
("realtor-listing", "realtor.com", "real_estate"),
("reddit-post", "reddit.com", "social"),
("rei-product", "rei.com", "ecommerce"),
("reuters-article", "reuters.com", "news"),
("samsclub-product", "samsclub.com", "ecommerce"),
("scmp-article", "scmp.com", "news"),
("sec-edgar-filing", "sec.gov", "financial"),
("sephora-product", "sephora.com", "ecommerce"),
("staples-product", "staples.com", "ecommerce"),
("substack-article", "substack.com", "content"),
("target-product", "target.com", "ecommerce"),
("techcrunch-article", "techcrunch.com", "news"),
("theguardian-article", "theguardian.com", "news"),
("theverge-article", "theverge.com", "news"),
("tiktok-profile", "tiktok.com", "social"),
("time-article", "time.com", "news"),
("tripadvisor-attraction", "tripadvisor.com", "travel"),
("trustpilot-reviews", "trustpilot.com", "reviews"),
("twitter-profile", "twitter.com", "social"),
("ulta-product", "ulta.com", "ecommerce"),
("usatoday-article", "usatoday.com", "news"),
("uspto-patent", "uspto.gov", "reference"),
("walgreens-product", "walgreens.com", "ecommerce"),
("walmart-grocery-product", "walmart.com", "ecommerce"),
("walmart-product", "walmart.com", "ecommerce"),
("washingtonpost-article", "washingtonpost.com", "news"),
("wayfair-product", "wayfair.com", "ecommerce"),
("west-elm-product", "westelm.com", "ecommerce"),
("wikipedia-article", "wikipedia.org", "reference"),
("wired-article", "wired.com", "news"),
("wsj-article", "wsj.com", "news"),
("yahoo-finance-quote", "finance.yahoo.com", "financial"),
("yelp-business", "yelp.com", "reviews"),
("youtube-video", "youtube.com", "media"),
("zara-product", "zara.com", "ecommerce"),
("zerohedge-article", "zerohedge.com", "news"),
("zillow-listing", "zillow.com", "real_estate"),
("ziprecruiter-job", "ziprecruiter.com", "professional"),
]
async def test_all():
failures = []
async with httpx.AsyncClient(timeout=30) as c:
for tid, site, cat in TEMPLATES:
print(f"Testing {tid}...", end=" ", flush=True)
try:
r = await c.post(
"http://localhost:8002/v1/templates/execute",
json={"template_id": tid, "url": f"https://www.{site}"},
timeout=30,
)
if r.is_success and r.json().get("success"):
print("OK")
else:
print(f"FAIL: {r.text[:80]}")
failures.append(tid)
await asyncio.sleep(0.5)
except (httpx.HTTPError, httpx.RequestError) as e:
print(f"ERR: {e}")
failures.append(tid)
print(f"\n{len([t for t in TEMPLATES])} tested, {len(failures)} failed")
if failures:
print("Failed:", ", ".join(failures))
return 1 if failures else 0
if __name__ == "__main__":
sys.exit(asyncio.run(test_all()))
Reference in a new issue View git blame Copy permalink