cryptorugmunch
47ba268131
Adds missing standard artifacts: - README.md (if missing) - AGENTS.md (AI agent contract) - PLAN.md (current sprint) - STATUS.md (where we are) - DEVELOPMENT.md (dev workflow) - DEPLOYMENT.md (deploy procedure) - TESTING.md (test strategy) - DECISIONS.md (ADR index + templates) - .github/CODEOWNERS - .github/workflows/ci.yml Preserves all existing artifacts. Refs: RugMunchMedia/fleet-template
119 lines
3.6 KiB
Python
119 lines
3.6 KiB
Python
"""Pry — One-file config system.
|
|
Define all your scraping jobs in a pry.yml file, run them with `pry run`.
|
|
|
|
Example pry.yml:
|
|
```yaml
|
|
jobs:
|
|
- name: product_prices
|
|
url: https://store.com/products
|
|
schedule: every 1h
|
|
extract:
|
|
title: "h1.product-name"
|
|
price: ".price"
|
|
stock: ".stock-status"
|
|
output: csv
|
|
webhook: slack://C012345
|
|
```
|
|
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
|
|
import yaml
|
|
|
|
|
|
class Pryfile:
|
|
"""Parse and execute pry.yml job definitions."""
|
|
|
|
def __init__(self, path: str = "pry.yml"):
|
|
self.path = path
|
|
self.jobs = []
|
|
self._load()
|
|
|
|
def _load(self):
|
|
if not os.path.exists(self.path):
|
|
# Try pry.yaml, Pryfile, pryfile.yml
|
|
for alt in ["pry.yaml", "Pryfile", "pryfile.yml", "pryfile.yaml"]:
|
|
if os.path.exists(alt):
|
|
self.path = alt
|
|
break
|
|
else:
|
|
return # No config file found — that's OK for CLI usage
|
|
with open(self.path) as f:
|
|
data = yaml.safe_load(f)
|
|
self.jobs = data.get("jobs", []) if data else []
|
|
self.global_settings = {k: v for k, v in (data or {}).items() if k != "jobs"}
|
|
|
|
async def run_all(self, scraper=None) -> list[dict]:
|
|
"""Execute all jobs defined in pry.yml. Returns results."""
|
|
from scraper import PryScraper
|
|
|
|
s = scraper or PryScraper()
|
|
results = []
|
|
for job in self.jobs:
|
|
try:
|
|
result = await self._run_job(job, s)
|
|
results.append(result)
|
|
except Exception as e:
|
|
results.append({"name": job.get("name", "unknown"), "error": str(e)})
|
|
return results
|
|
|
|
async def _run_job(self, job: dict, scraper) -> dict:
|
|
name = job.get("name", "unnamed")
|
|
url = job.get("url", "")
|
|
if not url:
|
|
return {"name": name, "error": "No URL specified"}
|
|
|
|
scrape_result = await scraper.scrape(
|
|
url,
|
|
{
|
|
"timeout": job.get("timeout", 30),
|
|
"bypass_cloudflare": job.get("bypass_cloudflare", True),
|
|
},
|
|
)
|
|
|
|
result = {
|
|
"name": name,
|
|
"url": url,
|
|
"status": scrape_result.get("status"),
|
|
"method": scrape_result.get("method", "unknown"),
|
|
"content_length": len(scrape_result.get("content", "")),
|
|
}
|
|
|
|
# Extract structured fields if defined
|
|
extract_schema = job.get("extract", {})
|
|
if extract_schema:
|
|
from extractor import SchemaExtractor
|
|
|
|
ex = SchemaExtractor()
|
|
extracted = ex._pattern_extract(scrape_result.get("content", ""), extract_schema)
|
|
result["extracted"] = extracted
|
|
|
|
# Transform output if specified
|
|
output_format = job.get("output", self.global_settings.get("output", "json"))
|
|
if output_format == "csv" and "extracted" in result:
|
|
import csv
|
|
import io
|
|
|
|
buf = io.StringIO()
|
|
w = csv.DictWriter(buf, fieldnames=result["extracted"].keys())
|
|
w.writeheader()
|
|
w.writerow(result["extracted"])
|
|
result["output"] = buf.getvalue()
|
|
elif output_format == "json":
|
|
result["output"] = json.dumps(result.get("extracted", {}), indent=2)
|
|
|
|
return result
|
|
|
|
def list_jobs(self) -> list[dict]:
|
|
"""List all configured jobs without executing them."""
|
|
return [
|
|
{
|
|
"name": j.get("name"),
|
|
"url": j.get("url"),
|
|
"schedule": j.get("schedule", "manual"),
|
|
"output": j.get("output", "json"),
|
|
}
|
|
for j in self.jobs
|
|
]
|