Anti Bot
Legal and Compliance Boundaries
Legal and Compliance Boundaries
This is not legal advice. Link to official APIs and alternative data, defending against scrapers, red teaming your own defenses, keeping current.
Jurisdiction-Specific Considerations
Laws vary by jurisdiction and facts matter.
Contracts and Terms
Understand contractual obligations before collecting data.
Circumvention
Consider applicable provisions carefully and document authorisation.
Privacy and Data Protection
Follow purpose limitation, minimisation, retention and deletion duties.
Intellectual Property
Distinguish facts from protected expression.
Abuse and Access
Respect technical controls and documented limits.
Documentation
Maintain authorisation, inventory, retention and deletion records.
import re
from collections import Counter
from datetime import date
# An internal operational policy, stated once and machine-checkable. This is
# a standard your team writes for itself; it is not a statement of what any
# jurisdiction requires, and no linter can decide that for you.
POLICY = {
"reference_date": date(2026, 10, 1),
"max_retention_days": 180,
"min_cell_size": 5,
"allowed_hosts": ("api.partner.example", "jobs.example.com",
"news.example.com", "shop.example.com"),
"allowed_purposes": ("academic research", "archive", "price monitoring",
"product analytics"),
"required_authorisation_for_gated": True,
}
# Field names that identify, or contribute to identifying, a person. A name
# match is a prompt to document a lawful basis, not an automatic deletion
# order: published professional information is still personal data.
PERSONAL = re.compile(
r"account_id|author|candidate|comment|dob|email|gender|ip_address|name|"
r"passport|phone|postcode|review_text|user_agent|username")
# One row per extract in the crawl manifest.
# id, host, purpose, authorisation reference or None, collection date,
# fields collected, subject records, auth-gated route, lawful basis or None,
# deletion path implemented
MANIFEST = [
("x001", "api.partner.example", "price monitoring", "AUTH-2026-0031",
date(2026, 9, 20), ["sku", "price", "stock_count", "published_at"],
2100000, False, None, True),
("x002", "shop.example.com", "price monitoring", "AUTH-2026-0031",
date(2026, 9, 29), ["sku", "price", "currency", "vendor_name"],
184000, False, None, True),
("x003", "shop.example.com", "product analytics", "AUTH-2026-0044",
date(2026, 6, 14), ["sku", "category", "spec_text", "image_url"],
41000, False, None, True),
("x004", "news.example.com", "archive", "AUTH-2026-0009",
date(2026, 1, 8), ["headline", "published_at", "author_name",
"comment_body"],
88000, False, None, False),
("x005", "forum.example.net", "archive", None,
date(2026, 8, 2), ["thread_title", "username", "comment_body"],
5100, False, None, True),
("x006", "jobs.example.com", "product analytics", None,
date(2026, 9, 11), ["job_title", "salary_band", "candidate_name",
"candidate_email"],
7400, False, None, False),
("x007", "jobs.example.com", "product analytics", "AUTH-2026-0052",
date(2026, 9, 30), ["job_title", "salary_band", "posting_date"],
6100, False, None, True),
("x008", "data.gov.example", "academic research", "AUTH-2026-0061",
date(2026, 9, 25), ["tender_title", "awarding_body", "award_value"],
9400, False, None, True),
("x009", "shop.example.com", "lead generation", "AUTH-2026-0044",
date(2026, 9, 18), ["buyer_name", "email", "phone", "postcode",
"ip_address"],
310000, False, "legitimate interest", False),
("x010", "shop.example.com", "academic research", "AUTH-2026-0061",
date(2026, 3, 3), ["review_text", "username", "rating", "posted_at"],
3, False, "public interest research", True),
("x011", "shop.example.com", "academic research", "AUTH-2026-0061",
date(2026, 9, 12), ["review_text", "rating", "posted_at"],
26000, False, "public interest research", True),
("x012", "api.partner.example", "price monitoring", "AUTH-2026-0031",
date(2026, 9, 30), ["account_id", "plan", "usage_count"],
1240, True, "contract", True),
("x013", "shop.example.com", "product analytics", "AUTH-2026-0044",
date(2026, 8, 21), ["sku", "price", "review_text", "buyer_name"],
182000, False, None, False),
("x014", "news.example.com", "academic research", "AUTH-2026-0061",
date(2026, 2, 14), ["headline", "published_at", "author_name"],
41000, False, None, False),
]
CODES = [
("E1", "personal data field with no documented lawful basis", 3),
("E2", "retention age beyond the policy window", 3),
("E3", "no authorisation reference on the manifest row", 2),
("E4", "host is not on the approved list", 3),
("E5", "purpose is not on the approved list", 2),
("E6", "auth-gated route collected without authorisation", 3),
("W1", "cell smaller than the re-identification floor", 1),
("W2", "personal data with no deletion path", 2),
]
SEVERITY = dict((c[0], c[2]) for c in CODES)
LABEL = dict((c[0], c[1]) for c in CODES)
def out(line=""):
print(line.rstrip())
def personal_fields(fields):
return sorted(f for f in fields if PERSONAL.search(f))
def lint(row):
(rid, host, purpose, auth, collected, fields, subjects, gated, basis,
deletion) = row
hits = []
pf = personal_fields(fields)
age = (POLICY["reference_date"] - collected).days
if host not in POLICY["allowed_hosts"]:
hits.append("E4")
if purpose not in POLICY["allowed_purposes"]:
hits.append("E5")
if auth is None:
hits.append("E3")
if gated and POLICY["required_authorisation_for_gated"] and auth is None:
hits.append("E6")
if pf and basis is None:
hits.append("E1")
if age > POLICY["max_retention_days"]:
hits.append("E2")
if pf and subjects < POLICY["min_cell_size"]:
hits.append("W1")
if pf and not deletion:
hits.append("W2")
return {"id": rid, "host": host, "purpose": purpose, "age": age,
"fields": len(fields), "personal": pf, "subjects": subjects,
"hits": sorted(hits, key=lambda c: (-SEVERITY[c], c)),
"basis": basis, "auth": auth, "deletion": deletion,
"collected": collected}
results = [lint(r) for r in MANIFEST]
errors = [r for r in results if [h for h in r["hits"] if h[0] == "E"]]
out("crawl manifest compliance lint")
out("policy window: max %d days retention, minimum cell %d, "
"gated routes need authorisation"
% (POLICY["max_retention_days"], POLICY["min_cell_size"]))
out("approved hosts: %s" % ", ".join(POLICY["allowed_hosts"]))
out("approved purposes: %s" % ", ".join(POLICY["allowed_purposes"]))
out("rows: %d records represented: %s reference date: %s"
% (len(results), format(sum(r["subjects"] for r in results), ","),
POLICY["reference_date"].isoformat()))
out()
out("%-5s %-24s %-18s %5s %8s %5s %s"
% ("row", "host", "purpose", "age", "records", "pii", "verdict"))
for r in results:
verdict = "clean" if not r["hits"] else " ".join(r["hits"])
out("%-5s %-24s %-18s %5d %8s %5d %s"
% (r["id"], r["host"], r["purpose"], r["age"],
format(r["subjects"], ","), len(r["personal"]), verdict))
counts = Counter(h for r in results for h in r["hits"])
out()
out("violations by rule")
for code, label, sev in sorted(CODES, key=lambda c: (-c[2], c[0])):
n = counts.get(code, 0)
out(" %-3s severity %d %-52s %d row%s"
% (code, sev, label, n, "" if n == 1 else "s"))
out()
out("records affected by each rule")
for code in sorted(counts):
touched = sum(r["subjects"] for r in results if code in r["hits"])
out(" %-3s %12s subject records across %d row(s)"
% (code, format(touched, ","),
len([r for r in results if code in r["hits"]])))
pii_rows = [r for r in results if r["personal"]]
all_pii = sorted(set(f for r in results for f in r["personal"]))
out()
out("personal data in the manifest: %d of %d rows, %d distinct fields"
% (len(pii_rows), len(results), len(all_pii)))
for f in all_pii:
rows = [r["id"] for r in results if f in r["personal"]]
basis = sorted(set(r["basis"] or "UNDOCUMENTED" for r in results
if f in r["personal"]))
out(" %-18s %s basis: %s" % (f, " ".join(rows), ", ".join(basis)))
out()
out("remediation queue, highest severity first")
for r in sorted(results, key=lambda r: r["id"]):
if not r["hits"]:
continue
todo = []
for h in r["hits"]:
if h == "E1":
todo.append("document a lawful basis or drop %s"
% ", ".join(r["personal"]))
elif h == "E2":
todo.append("purge or re-derive: %d days old"
% (r["age"] - POLICY["max_retention_days"]))
elif h in ("E3", "E6"):
todo.append("attach an authorisation reference to the row")
elif h == "E4":
todo.append("remove the host or get it approved")
elif h == "E5":
todo.append("re-file the purpose as one of: %s"
% ", ".join(POLICY["allowed_purposes"]))
elif h == "W1":
todo.append("suppress: %d subjects is under the floor"
% r["subjects"])
elif h == "W2":
todo.append("build the deletion path before the next run")
out(" %-5s %s" % (r["id"], "; ".join(todo)))
out()
out("%d of %d rows clean, %d need work before the next collection window"
% (len(results) - len(errors), len(results), len(errors)))
out("the linter checks the manifest against your stated policy. it cannot")
out("tell you whether the purpose is lawful or the basis is honest.")
crawl manifest compliance lint
policy window: max 180 days retention, minimum cell 5, gated routes need authorisation
approved hosts: api.partner.example, jobs.example.com, news.example.com, shop.example.com
approved purposes: academic research, archive, price monitoring, product analytics
rows: 14 records represented: 3,001,243 reference date: 2026-10-01
row host purpose age records pii verdict
x001 api.partner.example price monitoring 11 2,100,000 0 clean
x002 shop.example.com price monitoring 2 184,000 1 E1
x003 shop.example.com product analytics 109 41,000 0 clean
x004 news.example.com archive 266 88,000 2 E1 E2 W2
x005 forum.example.net archive 60 5,100 2 E1 E4 E3
x006 jobs.example.com product analytics 20 7,400 2 E1 E3 W2
x007 jobs.example.com product analytics 1 6,100 0 clean
x008 data.gov.example academic research 6 9,400 0 E4
x009 shop.example.com lead generation 13 310,000 5 E5 W2
x010 shop.example.com academic research 212 3 2 E2 W1
x011 shop.example.com academic research 19 26,000 1 clean
x012 api.partner.example price monitoring 1 1,240 1 clean
x013 shop.example.com product analytics 41 182,000 2 E1 W2
x014 news.example.com academic research 229 41,000 1 E1 E2 W2
violations by rule
E1 severity 3 personal data field with no documented lawful basis 6 rows
E2 severity 3 retention age beyond the policy window 3 rows
E4 severity 3 host is not on the approved list 2 rows
E6 severity 3 auth-gated route collected without authorisation 0 rows
E3 severity 2 no authorisation reference on the manifest row 2 rows
E5 severity 2 purpose is not on the approved list 1 row
W2 severity 2 personal data with no deletion path 5 rows
W1 severity 1 cell smaller than the re-identification floor 1 row
records affected by each rule
E1 507,500 subject records across 6 row(s)
E2 129,003 subject records across 3 row(s)
E3 12,500 subject records across 2 row(s)
E4 14,500 subject records across 2 row(s)
E5 310,000 subject records across 1 row(s)
W1 3 subject records across 1 row(s)
W2 628,400 subject records across 5 row(s)
personal data in the manifest: 10 of 14 rows, 13 distinct fields
account_id x012 basis: contract
author_name x004 x014 basis: UNDOCUMENTED
buyer_name x009 x013 basis: UNDOCUMENTED, legitimate interest
candidate_email x006 basis: UNDOCUMENTED
candidate_name x006 basis: UNDOCUMENTED
comment_body x004 x005 basis: UNDOCUMENTED
email x009 basis: legitimate interest
ip_address x009 basis: legitimate interest
phone x009 basis: legitimate interest
postcode x009 basis: legitimate interest
review_text x010 x011 x013 basis: UNDOCUMENTED, public interest research
username x005 x010 basis: UNDOCUMENTED, public interest research
vendor_name x002 basis: UNDOCUMENTED
remediation queue, highest severity first
x002 document a lawful basis or drop vendor_name
x004 document a lawful basis or drop author_name, comment_body; purge or re-derive: 86 days old; build the deletion path before the next run
x005 document a lawful basis or drop comment_body, username; remove the host or get it approved; attach an authorisation reference to the row
x006 document a lawful basis or drop candidate_email, candidate_name; attach an authorisation reference to the row; build the deletion path before the next run
x008 remove the host or get it approved
x009 re-file the purpose as one of: academic research, archive, price monitoring, product analytics; build the deletion path before the next run
x010 purge or re-derive: 32 days old; suppress: 3 subjects is under the floor
x013 document a lawful basis or drop buyer_name, review_text; build the deletion path before the next run
x014 document a lawful basis or drop author_name; purge or re-derive: 49 days old; build the deletion path before the next run
5 of 14 rows clean, 9 need work before the next collection window
the linter checks the manifest against your stated policy. it cannot
tell you whether the purpose is lawful or the basis is honest.
Lesson 60 of 62
See where this fits: Anti-Bot Evasion Roadmap
open roadmap ->