mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-16 14:32:34 +00:00
c2828f2b9b
unbroker finds where a consenting person's info is exposed across data brokers and people-search sites and files the removals, running as far as each site allows and handing only genuinely human-only steps (hard CAPTCHA, gov-ID, phone, fax) back as an end-of-run digest. - Deterministic stdlib CLI (scripts/pdd.py) owns config, dossiers+consent, the broker DB, tier planning, the ledger, email, and the autonomous action queue; the agent scans/submits with native tools (web_extract, browser_*, delegate_task, cronjob, terminal). - Verify-before-disclose, least-disclosure (never volunteers SSN), consent gate, opaque ids, optional age-at-rest encryption, file-locked ledger. - Jurisdiction-aware (CCPA/CPRA, GDPR, generic); CA DROP one-shot covers the state registry (~545) in a single request; BADBOOL + curated people-search coverage; scheduled re-scan for re-listing. - No CAPTCHA-solving services or anti-bot bypass; browser email mode needs no stored password. - 85 hermetic tests (tests/skills/test_unbroker_skill.py; SMTP/IMAP via injected fakes, registry via CSV fixtures). Ships placeholder data only. Broker dataset adapted from BADBOOL (Yael Grauer, CC BY-NC-SA 4.0).
77 lines
2.6 KiB
Python
77 lines
2.6 KiB
Python
"""Load and query the broker database (references/brokers/*.json).
|
|
|
|
Each broker is one JSON file for clean diffs/PRs. Files beginning with `_` are
|
|
ignored (reserved for notes/scratch).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import paths
|
|
import storage
|
|
|
|
PRIORITY_ORDER = {"crucial": 0, "high": 1, "standard": 2, "long_tail": 3}
|
|
|
|
|
|
def _load_curated(directory: Path | None = None) -> list[dict]:
|
|
directory = directory or paths.brokers_dir()
|
|
out: list[dict] = []
|
|
if not directory.exists():
|
|
return out
|
|
for fp in sorted(directory.glob("*.json")):
|
|
if fp.name.startswith("_"):
|
|
continue
|
|
out.append(json.loads(fp.read_text(encoding="utf-8")))
|
|
return out
|
|
|
|
|
|
def load_live_cache() -> list[dict]:
|
|
"""Records pulled from BADBOOL via `refresh-brokers` (empty until refreshed)."""
|
|
return storage.read_json(paths.brokers_cache_path(), []) or []
|
|
|
|
|
|
def load_registry_cache() -> list[dict]:
|
|
"""CA Data Broker Registry records (separate coverage lane; empty until refreshed).
|
|
|
|
Kept OUT of load_all() by default: these are not people-search sites to scan, they
|
|
are worked via the CA DROP one-shot + CCPA email. Consumers of the scan/plan/fanout
|
|
pipeline must not receive them; use this directly for coverage counts and the DROP/
|
|
email lanes.
|
|
"""
|
|
return storage.read_json(paths.registry_cache_path(), []) or []
|
|
|
|
|
|
def load_all(directory: Path | None = None, include_live: bool = True) -> list[dict]:
|
|
"""Curated records, with live BADBOOL records merged underneath (curated wins)."""
|
|
merged: dict[str, dict] = {b["id"]: b for b in _load_curated(directory)}
|
|
if include_live:
|
|
for b in load_live_cache():
|
|
bid = b.get("id")
|
|
if bid and bid not in merged:
|
|
merged[bid] = b
|
|
out = list(merged.values())
|
|
out.sort(key=lambda b: (PRIORITY_ORDER.get(b.get("priority", "standard"), 9), b.get("id", "")))
|
|
return out
|
|
|
|
|
|
def get(broker_id: str, directory: Path | None = None) -> dict | None:
|
|
for b in load_all(directory):
|
|
if b.get("id") == broker_id:
|
|
return b
|
|
return None
|
|
|
|
|
|
def by_priority(*levels: str, directory: Path | None = None) -> list[dict]:
|
|
wanted = set(levels) if levels else None
|
|
return [b for b in load_all(directory) if wanted is None or b.get("priority") in wanted]
|
|
|
|
|
|
def clusters(directory: Path | None = None) -> dict[str, list[str]]:
|
|
"""Map a parent broker id -> child site ids it can clear (force-multipliers)."""
|
|
out: dict[str, list[str]] = {}
|
|
for b in load_all(directory):
|
|
owns = b.get("owns") or []
|
|
if owns:
|
|
out[b["id"]] = list(owns)
|
|
return out
|