docs: apply fleet-template (16-artifact scaffold)
Adds missing standard artifacts: - README.md (if missing) - AGENTS.md (AI agent contract) - PLAN.md (current sprint) - STATUS.md (where we are) - DEVELOPMENT.md (dev workflow) - DEPLOYMENT.md (deploy procedure) - TESTING.md (test strategy) - DECISIONS.md (ADR index + templates) - .github/CODEOWNERS - .github/workflows/ci.yml Preserves all existing artifacts. Refs: RugMunchMedia/fleet-template
This commit is contained in:
commit
47ba268131
310 changed files with 38429 additions and 0 deletions
119
pryfile.py
Normal file
119
pryfile.py
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
"""Pry — One-file config system.
|
||||
Define all your scraping jobs in a pry.yml file, run them with `pry run`.
|
||||
|
||||
Example pry.yml:
|
||||
```yaml
|
||||
jobs:
|
||||
- name: product_prices
|
||||
url: https://store.com/products
|
||||
schedule: every 1h
|
||||
extract:
|
||||
title: "h1.product-name"
|
||||
price: ".price"
|
||||
stock: ".stock-status"
|
||||
output: csv
|
||||
webhook: slack://C012345
|
||||
```
|
||||
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
class Pryfile:
|
||||
"""Parse and execute pry.yml job definitions."""
|
||||
|
||||
def __init__(self, path: str = "pry.yml"):
|
||||
self.path = path
|
||||
self.jobs = []
|
||||
self._load()
|
||||
|
||||
def _load(self):
|
||||
if not os.path.exists(self.path):
|
||||
# Try pry.yaml, Pryfile, pryfile.yml
|
||||
for alt in ["pry.yaml", "Pryfile", "pryfile.yml", "pryfile.yaml"]:
|
||||
if os.path.exists(alt):
|
||||
self.path = alt
|
||||
break
|
||||
else:
|
||||
return # No config file found — that's OK for CLI usage
|
||||
with open(self.path) as f:
|
||||
data = yaml.safe_load(f)
|
||||
self.jobs = data.get("jobs", []) if data else []
|
||||
self.global_settings = {k: v for k, v in (data or {}).items() if k != "jobs"}
|
||||
|
||||
async def run_all(self, scraper=None) -> list[dict]:
|
||||
"""Execute all jobs defined in pry.yml. Returns results."""
|
||||
from scraper import PryScraper
|
||||
|
||||
s = scraper or PryScraper()
|
||||
results = []
|
||||
for job in self.jobs:
|
||||
try:
|
||||
result = await self._run_job(job, s)
|
||||
results.append(result)
|
||||
except Exception as e:
|
||||
results.append({"name": job.get("name", "unknown"), "error": str(e)})
|
||||
return results
|
||||
|
||||
async def _run_job(self, job: dict, scraper) -> dict:
|
||||
name = job.get("name", "unnamed")
|
||||
url = job.get("url", "")
|
||||
if not url:
|
||||
return {"name": name, "error": "No URL specified"}
|
||||
|
||||
scrape_result = await scraper.scrape(
|
||||
url,
|
||||
{
|
||||
"timeout": job.get("timeout", 30),
|
||||
"bypass_cloudflare": job.get("bypass_cloudflare", True),
|
||||
},
|
||||
)
|
||||
|
||||
result = {
|
||||
"name": name,
|
||||
"url": url,
|
||||
"status": scrape_result.get("status"),
|
||||
"method": scrape_result.get("method", "unknown"),
|
||||
"content_length": len(scrape_result.get("content", "")),
|
||||
}
|
||||
|
||||
# Extract structured fields if defined
|
||||
extract_schema = job.get("extract", {})
|
||||
if extract_schema:
|
||||
from extractor import SchemaExtractor
|
||||
|
||||
ex = SchemaExtractor()
|
||||
extracted = ex._pattern_extract(scrape_result.get("content", ""), extract_schema)
|
||||
result["extracted"] = extracted
|
||||
|
||||
# Transform output if specified
|
||||
output_format = job.get("output", self.global_settings.get("output", "json"))
|
||||
if output_format == "csv" and "extracted" in result:
|
||||
import csv
|
||||
import io
|
||||
|
||||
buf = io.StringIO()
|
||||
w = csv.DictWriter(buf, fieldnames=result["extracted"].keys())
|
||||
w.writeheader()
|
||||
w.writerow(result["extracted"])
|
||||
result["output"] = buf.getvalue()
|
||||
elif output_format == "json":
|
||||
result["output"] = json.dumps(result.get("extracted", {}), indent=2)
|
||||
|
||||
return result
|
||||
|
||||
def list_jobs(self) -> list[dict]:
|
||||
"""List all configured jobs without executing them."""
|
||||
return [
|
||||
{
|
||||
"name": j.get("name"),
|
||||
"url": j.get("url"),
|
||||
"schedule": j.get("schedule", "manual"),
|
||||
"output": j.get("output", "json"),
|
||||
}
|
||||
for j in self.jobs
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue