pryscraper/crm_sync.py

"""Pry — Reverse ETL to CRM.
Sync scraped data to Salesforce, HubSpot, Pipedrive, and Close.com."""

# SPDX-License-Identifier: MIT
# Copyright (c) 2026 Rug Munch Media LLC
#
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
# Licensed under MIT. See LICENSE.

import logging
from typing import Any

logger = logging.getLogger(__name__)


async def sync_to_salesforce(
    objects: list[dict[str, Any]],
    object_type: str = "Lead",
    instance_url: str = "",
    access_token: str = "",
) -> dict[str, Any]:
    """Sync scraped data to Salesforce objects.

    Args:
        objects: List of records to create/update
        object_type: Salesforce object type (Lead, Contact, Account, Opportunity)
        instance_url: Salesforce instance URL (e.g., https://yourInstance.salesforce.com)
        access_token: Salesforce OAuth2 access token
    """
    from client import get_client

    client = await get_client()
    api_url = instance_url.rstrip("/")

    results = []
    for obj in objects:
        sf_obj = _map_to_salesforce(obj, object_type)
        try:
            resp = await client.post(
                f"{api_url}/services/data/v58.0/sobjects/{object_type}",
                json=sf_obj,
                headers={
                    "Authorization": f"Bearer {access_token}",
                    "Content-Type": "application/json",
                },
                timeout=30,
            )
            if resp.is_success:
                data = resp.json()
                results.append(
                    {
                        "success": True,
                        "salesforce_id": data.get("id"),
                        "name": obj.get("name") or obj.get("title", "Unknown"),
                    }
                )
            else:
                results.append(
                    {
                        "success": False,
                        "name": obj.get("name", "Unknown"),
                        "error": f"Salesforce error {resp.status_code}: {resp.text[:200]}",
                    }
                )
        except Exception as e:
            results.append(
                {
                    "success": False,
                    "name": obj.get("name", "Unknown"),
                    "error": str(e)[:200],
                }
            )

    success_count = sum(1 for r in results if r["success"])
    return {
        "success": success_count > 0,
        "platform": "salesforce",
        "object_type": object_type,
        "total": len(objects),
        "synced": success_count,
        "failed": len(objects) - success_count,
        "results": results,
    }


def _map_to_salesforce(data: dict[str, Any], object_type: str) -> dict[str, Any]:
    """Map common field names to Salesforce standard field names."""
    mapping = {
        "name": "LastName" if object_type == "Lead" else "Name",
        "first_name": "FirstName",
        "last_name": "LastName",
        "email": "Email",
        "phone": "Phone",
        "company": "Company",
        "title": "Title",
        "description": "Description",
        "website": "Website",
        "industry": "Industry",
        "address": "Street",
        "city": "City",
        "state": "State",
        "zip": "PostalCode",
        "country": "Country",
        "revenue": "AnnualRevenue",
        "employees": "NumberOfEmployees",
        "source_url": "LeadSource",
    }

    result = {}
    for src_key, sf_key in mapping.items():
        if data.get(src_key):
            result[sf_key] = data[src_key]

    for key, value in data.items():
        if key not in mapping and key not in ("_source", "_raw"):
            result[key[:120]] = (
                value if isinstance(value, (str, int, float, bool)) else str(value)[:255]
            )

    return result


async def sync_to_hubspot(
    objects: list[dict[str, Any]],
    object_type: str = "contacts",
    api_key: str = "",
) -> dict[str, Any]:
    """Sync scraped data to HubSpot CRM.

    Args:
        objects: List of contact/company/deal records
        object_type: contacts, companies, deals
        api_key: HubSpot Private App API key
    """
    from client import get_client

    client = await get_client()
    headers = {
        "Authorization": f"Bearer {api_key}",
        "Content-Type": "application/json",
    }
    results = []

    api_paths = {
        "contacts": "/crm/v3/objects/contacts",
        "companies": "/crm/v3/objects/companies",
        "deals": "/crm/v3/objects/deals",
    }
    api_path = api_paths.get(object_type, f"/crm/v3/objects/{object_type}")

    for obj in objects:
        properties = _map_to_hubspot(obj, object_type)
        payload = {"properties": properties}
        try:
            resp = await client.post(
                f"https://api.hubapi.com{api_path}",
                json=payload,
                headers=headers,
                timeout=30,
            )
            if resp.is_success:
                data = resp.json()
                results.append(
                    {
                        "success": True,
                        "hubspot_id": data.get("id"),
                        "name": obj.get("name") or obj.get("email", "Unknown"),
                    }
                )
            else:
                results.append(
                    {
                        "success": False,
                        "name": obj.get("name", "Unknown"),
                        "error": f"HubSpot error {resp.status_code}: {resp.text[:200]}",
                    }
                )
        except Exception as e:
            results.append(
                {
                    "success": False,
                    "name": obj.get("name", "Unknown"),
                    "error": str(e)[:200],
                }
            )

    success_count = sum(1 for r in results if r["success"])
    return {
        "success": success_count > 0,
        "platform": "hubspot",
        "object_type": object_type,
        "total": len(objects),
        "synced": success_count,
        "failed": len(objects) - success_count,
        "results": results,
    }


def _map_to_hubspot(data: dict[str, Any], object_type: str) -> dict[str, str]:
    """Map common fields to HubSpot property names."""
    mapping: dict[str, str] = {
        "name": "firstname" if object_type == "contacts" else "name",
        "first_name": "firstname",
        "last_name": "lastname",
        "email": "email",
        "phone": "phone",
        "company": "company",
        "title": "jobtitle",
        "website": "website",
        "description": "description",
        "address": "address",
        "city": "city",
        "state": "state",
        "zip": "zip",
        "country": "country",
        "industry": "industry",
    }
    properties = {}
    for src_key, hs_key in mapping.items():
        if data.get(src_key):
            properties[hs_key] = str(data[src_key])[:500]
    return properties


async def sync_to_pipedrive(
    objects: list[dict[str, Any]],
    object_type: str = "person",
    api_token: str = "",
    domain: str = "",
) -> dict[str, Any]:
    """Sync scraped data to Pipedrive CRM."""
    from client import get_client

    client = await get_client()
    domain = domain or "mycompany"
    results = []

    api_paths = {
        "person": "/v1/persons",
        "organization": "/v1/organizations",
        "deal": "/v1/deals",
        "lead": "/v1/leads",
    }
    path = api_paths.get(object_type, "/v1/persons")
    base = f"https://{domain}.pipedrive.com/api{path}"

    for obj in objects:
        fields = _map_to_pipedrive(obj, object_type)
        fields["api_token"] = api_token
        try:
            resp = await client.post(base, json=fields, timeout=30)
            if resp.is_success:
                data = resp.json()
                results.append(
                    {
                        "success": True,
                        "pipedrive_id": data.get("data", {}).get("id"),
                        "name": obj.get("name", "Unknown"),
                    }
                )
            else:
                results.append(
                    {
                        "success": False,
                        "name": obj.get("name", "Unknown"),
                        "error": f"Pipedrive error {resp.status_code}",
                    }
                )
        except Exception as e:
            results.append(
                {
                    "success": False,
                    "name": obj.get("name", "Unknown"),
                    "error": str(e)[:200],
                }
            )

    success_count = sum(1 for r in results if r["success"])
    return {
        "success": success_count > 0,
        "platform": "pipedrive",
        "total": len(objects),
        "synced": success_count,
        "failed": len(objects) - success_count,
        "results": results,
    }


def _map_to_pipedrive(data: dict[str, Any], object_type: str) -> dict[str, Any]:
    mapping = {
        "name": "name",
        "email": "email",
        "phone": "phone",
        "company": "org_name",
        "title": "title",
    }
    result: dict[str, Any] = {}
    for src_key, pd_key in mapping.items():
        if data.get(src_key):
            result[pd_key] = str(data[src_key])[:500]
    if object_type == "person" and "email" in result:
        result["email"] = [{"value": result["email"], "primary": True}]
    return result


async def sync_to_close(
    objects: list[dict[str, Any]],
    object_type: str = "lead",
    api_key: str = "",
) -> dict[str, Any]:
    """Sync scraped data to Close.com CRM."""
    import base64

    from client import get_client

    client = await get_client()
    auth = base64.b64encode(f"{api_key}:".encode()).decode()
    results = []

    for obj in objects:
        close_obj = {
            "name": obj.get("name") or obj.get("title", "Unknown"),
            "description": (obj.get("description") or obj.get("content", ""))[:500],
            "url": obj.get("url", ""),
        }
        if obj.get("email"):
            close_obj["contacts"] = [{"emails": [{"email": obj["email"]}]}]
        try:
            resp = await client.post(
                "https://api.close.com/api/v1/lead/",
                json=close_obj,
                headers={"Authorization": f"Basic {auth}", "Content-Type": "application/json"},
                timeout=30,
            )
            if resp.is_success:
                data = resp.json()
                results.append(
                    {
                        "success": True,
                        "close_id": data.get("id"),
                        "name": obj.get("name", "Unknown"),
                    }
                )
            else:
                results.append(
                    {
                        "success": False,
                        "name": obj.get("name", "Unknown"),
                        "error": f"Close error {resp.status_code}: {resp.text[:200]}",
                    }
                )
        except Exception as e:
            results.append(
                {"success": False, "name": obj.get("name", "Unknown"), "error": str(e)[:200]}
            )

    success_count = sum(1 for r in results if r["success"])
    return {
        "success": success_count > 0,
        "platform": "close",
        "total": len(objects),
        "synced": success_count,
        "failed": len(objects) - success_count,
        "results": results,
    }