commit 41c57f6cd276d4edd598eec457a40f5df689a7fc Author: Théo Gastinel Date: Sat Oct 24 15:21:38 2026 +0200 Prototype mesure d'audience par UID localStorage (collecteur, snippet, agrégateur, tests) Co-Authored-By: Claude Sonnet 5.5 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b754c22 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.sqlite3* +*.json +.pytest_cache/ diff --git a/aggregate.py b/aggregate.py new file mode 100644 index 0000000..699cd1e --- /dev/null +++ b/aggregate.py @@ -0,0 +1,49 @@ +"""Agrégation quotidienne : SQLite -> bloc JSON pour data.json (stats.gastinel.me). + +Purge : lignes brutes > RAW_DAYS jours, sels des mois révolus (les identifiants +des mois passés deviennent alors définitivement non reliables). +Les agrégats (des compteurs, sans identifiant) peuvent être gardés 25 mois. +""" +import argparse, json, sqlite3, time +from collector import open_db + +RAW_DAYS = 62 # couvre le mois en cours + le précédent (visiteurs uniques mensuels) + + +def purge(db, now=None): + now = time.time() if now is None else now + cutoff = time.strftime("%Y-%m-%d", time.gmtime(now - RAW_DAYS * 86400)) + db.execute("DELETE FROM hits WHERE day < ?", (cutoff,)) + t = time.gmtime(now) + prev = f"{t.tm_year - (t.tm_mon == 1)}-{(t.tm_mon - 2) % 12 + 1:02d}" + db.execute("DELETE FROM salts WHERE month < ?", (prev,)) + + +def aggregate(db): + q = lambda sql: db.execute(sql).fetchall() + return { + "methode": "uid-localstorage-v1", + "jours": {d: {"visiteurs": v, "pages_vues": p} for d, v, p in + q("SELECT day, COUNT(DISTINCT vid), COUNT(*) FROM hits GROUP BY day ORDER BY day")}, + "mois": {m: {"visiteurs": v, "pages_vues": p} for m, v, p in + q("SELECT substr(day,1,7) m, COUNT(DISTINCT vid), COUNT(*) FROM hits GROUP BY m ORDER BY m")}, + "sources": dict(q("SELECT src, COUNT(*) FROM hits GROUP BY src ORDER BY 2 DESC")), + "pages": dict(q("SELECT path, COUNT(*) FROM hits GROUP BY path ORDER BY 2 DESC LIMIT 20")), + "appareils": dict(q("SELECT dev, COUNT(DISTINCT vid) FROM hits GROUP BY dev")), + "navigateurs": dict(q("SELECT nav, COUNT(DISTINCT vid) FROM hits GROUP BY nav")), + } + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--db", default="audience.sqlite3") + ap.add_argument("--out", default="audience.json") + a = ap.parse_args() + db = open_db(a.db) + purge(db) + with open(a.out, "w", encoding="utf-8") as f: + json.dump(aggregate(db), f, ensure_ascii=False, indent=1) + + +if __name__ == "__main__": + main() diff --git a/collector.py b/collector.py new file mode 100644 index 0000000..2e498e0 --- /dev/null +++ b/collector.py @@ -0,0 +1,193 @@ +"""Collecteur de mesure d'audience (stdlib uniquement). + +POST /m -> 204, corps JSON <= 512 o envoyé par navigator.sendBeacon (text/plain). +Aucune IP, aucun User-Agent complet n'est stocké : seulement famille de navigateur +grossière et type d'appareil. L'UID est pseudonymisé par HMAC avec un sel mensuel +(on ne peut pas relier un appareil d'un mois sur l'autre, ni retrouver l'UID d'origine). +""" +import argparse, hashlib, hmac, json, os, re, secrets, sqlite3, threading, time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlsplit + +MAX_BODY = 512 +UID_RE = re.compile(r"^[0-9a-f]{32}$") +PATH_RE = re.compile(r"^/[A-Za-z0-9._~/%-]{0,199}$") +HOST_RE = re.compile(r"^[a-z0-9.-]{0,100}$") +BOT_RE = re.compile(r"bot|crawl|spider|slurp|headless|phantom|python|curl|wget|httpclient|java/|go-http|scrapy|lighthouse|preview", re.I) +PER_UID_PER_MIN = 30 +GLOBAL_PER_SEC = 50 +OWN_HOSTS = ("gastinel.me",) +SEARCH = ("google.", "bing.", "duckduckgo.", "qwant.", "ecosia.", "yahoo.", "startpage.", "brave.") +SOCIAL = ("youtube.", "youtu.be", "instagram.", "facebook.", "t.co", "twitter.", "x.com", "tiktok.", "linkedin.", "reddit.", "discord", "whatsapp", "snapchat") + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS hits( + day TEXT NOT NULL, -- AAAA-MM-JJ (UTC), pas d'heure précise + vid TEXT NOT NULL, -- HMAC(uid, sel du mois), 16 hex + path TEXT NOT NULL, + src TEXT NOT NULL, -- direct|interne|recherche|social|autre + dev TEXT NOT NULL, -- mobile|ordinateur + nav TEXT NOT NULL -- firefox|chrome|safari|edge|autre +); +CREATE INDEX IF NOT EXISTS hits_day ON hits(day); +CREATE TABLE IF NOT EXISTS salts(month TEXT PRIMARY KEY, salt BLOB NOT NULL); +""" + + +def open_db(path): + db = sqlite3.connect(path, check_same_thread=False, isolation_level=None) + db.execute("PRAGMA journal_mode=WAL") + db.executescript(SCHEMA) + return db + + +def source_of(ref_host): + h = (ref_host or "").lower() + if not h: + return "direct" + if any(h == o or h.endswith("." + o) for o in OWN_HOSTS): + return "interne" + if any(s in h for s in SEARCH): + return "recherche" + if any(s in h for s in SOCIAL): + return "social" + return "autre" + + +def browser_of(ua): + u = ua or "" + if "Firefox/" in u: return "firefox" + if "Edg/" in u: return "edge" + if "Chrome/" in u or "CriOS" in u: return "chrome" + if "Safari/" in u: return "safari" + return "autre" + + +class Limiter: + def __init__(self): + self.lock = threading.Lock() + self.per_uid = {} # uid -> (minute, count) + self.sec = (0, 0) + + def allow(self, uid, now=None): + now = time.time() if now is None else now + m, s = int(now // 60), int(now) + with self.lock: + if self.sec[0] != s: + self.sec = (s, 0) + if self.sec[1] >= GLOBAL_PER_SEC: + return False + mm, c = self.per_uid.get(uid, (m, 0)) + if mm != m: + c = 0 + if len(self.per_uid) > 50000: + self.per_uid.clear() + if c >= PER_UID_PER_MIN: + return False + self.per_uid[uid] = (m, c + 1) + self.sec = (s, self.sec[1] + 1) + return True + + +class Store: + def __init__(self, db): + self.db, self.lock, self.limiter = db, threading.Lock(), Limiter() + + def salt(self, month): + row = self.db.execute("SELECT salt FROM salts WHERE month=?", (month,)).fetchone() + if row: + return row[0] + s = secrets.token_bytes(32) + self.db.execute("INSERT OR IGNORE INTO salts VALUES(?,?)", (month, s)) + return self.db.execute("SELECT salt FROM salts WHERE month=?", (month,)).fetchone()[0] + + def record(self, raw, ua="", origin_ok=True, now=None): + """Retourne True si l'événement est enregistré. Ne lève jamais.""" + if not origin_ok or len(raw) > MAX_BODY or BOT_RE.search(ua or "") or not ua: + return False + try: + ev = json.loads(raw.decode("utf-8")) + except Exception: + return False + if not isinstance(ev, dict) or set(ev) - {"u", "p", "r", "m"}: + return False + uid, path, ref, mob = ev.get("u"), ev.get("p"), ev.get("r", ""), ev.get("m", 0) + if not (isinstance(uid, str) and UID_RE.match(uid)): + return False + if not (isinstance(path, str) and PATH_RE.match(path)): + return False + if not (isinstance(ref, str) and HOST_RE.match(ref)) or mob not in (0, 1): + return False + if not self.limiter.allow(uid, now): + return False + t = time.gmtime(time.time() if now is None else now) + day, month = time.strftime("%Y-%m-%d", t), time.strftime("%Y-%m", t) + with self.lock: + vid = hmac.new(self.salt(month), uid.encode(), hashlib.sha256).hexdigest()[:16] + self.db.execute("INSERT INTO hits VALUES(?,?,?,?,?,?)", + (day, vid, path, source_of(ref), "mobile" if mob else "ordinateur", browser_of(ua))) + return True + + +def make_handler(store, allowed_origins, demo_dir=None): + class H(BaseHTTPRequestHandler): + server_version = "m" + sys_version = "" + + def log_message(self, *a): # pas de journal d'accès (il contiendrait l'IP) + pass + + def _send(self, code, body=b"", ctype="text/plain"): + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + if body: + self.wfile.write(body) + + def do_POST(self): + if urlsplit(self.path).path != "/m": + return self._send(404) + try: + n = int(self.headers.get("Content-Length", "0")) + except ValueError: + return self._send(400) + if n <= 0 or n > MAX_BODY: + return self._send(413 if n > MAX_BODY else 400) + raw = self.rfile.read(n) + org = self.headers.get("Origin", "") + ok = org in allowed_origins and self.headers.get("Sec-Fetch-Site", "same-origin") == "same-origin" + store.record(raw, self.headers.get("User-Agent", ""), ok) + self._send(204) # toujours 204 : ne rien apprendre aux robots + + def do_GET(self): + if demo_dir is None: + return self._send(404) + p = urlsplit(self.path).path + name = {"/": "index.html", "/index.html": "index.html", "/page2.html": "page2.html", "/m.js": "m.js"}.get(p) + if not name: + return self._send(404) + with open(os.path.join(demo_dir, name), "rb") as f: + data = f.read() + self._send(200, data, "application/javascript" if name.endswith(".js") else "text/html; charset=utf-8") + return H + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--host", default="127.0.0.1") + ap.add_argument("--port", type=int, default=8097) + ap.add_argument("--db", default="audience.sqlite3") + ap.add_argument("--origin", action="append", default=[]) + ap.add_argument("--demo", action="store_true", help="sert aussi demo/ (test local uniquement)") + a = ap.parse_args() + origins = set(a.origin or ["https://gastinel.me"]) + demo = os.path.join(os.path.dirname(os.path.abspath(__file__)), "demo") if a.demo else None + srv = ThreadingHTTPServer((a.host, a.port), make_handler(Store(open_db(a.db)), origins, demo)) + print(f"collecteur sur http://{a.host}:{a.port} pid={os.getpid()}", flush=True) + srv.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/demo/index.html b/demo/index.html new file mode 100644 index 0000000..d1ed0e3 --- /dev/null +++ b/demo/index.html @@ -0,0 +1,19 @@ + + +Démo mesure d'audience + + + +

Démo mesure d'audience (local)

+

Chaque chargement envoie un événement à /m. Page 2

+

UID stocké : ?

+ + + diff --git a/demo/m.js b/demo/m.js new file mode 100644 index 0000000..ffb835d --- /dev/null +++ b/demo/m.js @@ -0,0 +1,24 @@ +/* Mesure d'audience gastinel.me — UID aléatoire en localStorage, 13 mois max, sans prolongation. */ +(function (w, n, d) { + var K = "gm_uid", O = "gm_optout", MAX = 395 * 864e5, ls; + try { ls = w.localStorage; } catch (e) { return; } + w.gmOptOut = function (on) { + try { if (on === false) ls.removeItem(O); else { ls.setItem(O, "1"); ls.removeItem(K); } } catch (e) {} + }; + if (n.doNotTrack == "1" || w.doNotTrack == "1" || n.globalPrivacyControl === true) return; + try { if (ls.getItem(O) == "1") return; } catch (e) { return; } + var v, now = Date.now(); + try { v = JSON.parse(ls.getItem(K) || "null"); } catch (e) { v = null; } + if (!v || !/^[0-9a-f]{32}$/.test(v.id) || !(now - v.c < MAX) || v.c > now + 864e5) { + var b = new Uint8Array(16), h = ""; + w.crypto.getRandomValues(b); + for (var i = 0; i < 16; i++) h += (b[i] + 256).toString(16).slice(1); + v = { id: h, c: now }; + try { ls.setItem(K, JSON.stringify(v)); } catch (e) { return; } + } + var r = ""; + try { r = d.referrer ? new URL(d.referrer).hostname : ""; } catch (e) {} + var body = JSON.stringify({ u: v.id, p: location.pathname.slice(0, 200), r: r.slice(0, 100), + m: w.matchMedia && w.matchMedia("(pointer:coarse)").matches ? 1 : 0 }); + if (n.sendBeacon) n.sendBeacon("/m", body); +})(window, navigator, document); diff --git a/demo/page2.html b/demo/page2.html new file mode 100644 index 0000000..1d5fc8b --- /dev/null +++ b/demo/page2.html @@ -0,0 +1,3 @@ + +Démo page 2 +

Page 2

Retour

diff --git a/tests/test_audience.py b/tests/test_audience.py new file mode 100644 index 0000000..f67ca54 --- /dev/null +++ b/tests/test_audience.py @@ -0,0 +1,122 @@ +import json, os, sys, threading, time, urllib.request, urllib.error +from http.server import ThreadingHTTPServer +import pytest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import collector as c +import aggregate as a + +UA = "Mozilla/5.0 (X11; Linux x86_64; rv:130.0) Gecko/20100101 Firefox/130.0" +UID = "0123456789abcdef0123456789abcdef" + + +def ev(**k): + d = {"u": UID, "p": "/", "r": "", "m": 0} + d.update(k) + return json.dumps(d).encode() + + +@pytest.fixture +def store(tmp_path): + return c.Store(c.open_db(str(tmp_path / "t.db"))) + + +def test_uid_valide(store): + assert store.record(ev(), UA) + + +@pytest.mark.parametrize("uid", ["", "xyz", UID.upper(), UID + "0", UID[:-1], 123, None, "g" * 32]) +def test_uid_invalide(store, uid): + assert not store.record(ev(u=uid), UA) + + +@pytest.mark.parametrize("bad", [{"p": "/a?q=email@x.fr"}, {"p": "http://x"}, {"p": "/" + "a" * 300}, + {"r": "Evil Host"}, {"m": 2}, {"x": 1}]) +def test_champs_refuses(store, bad): + assert not store.record(ev(**bad), UA) + + +def test_taille_max(store): + assert not store.record(b"{" + b" " * c.MAX_BODY + b"}", UA) + assert not store.record(b"pas du json", UA) + + +def test_bots_et_ua_vide(store): + assert not store.record(ev(), "Googlebot/2.1") + assert not store.record(ev(), "HeadlessChrome/120") + assert not store.record(ev(), "") + + +def test_origine_refusee(store): + assert not store.record(ev(), UA, origin_ok=False) + + +def test_flood_par_uid(store): + now = 1_800_000_000 + ok = sum(store.record(ev(), UA, now=now) for _ in range(100)) + assert ok == c.PER_UID_PER_MIN + assert store.record(ev(), UA, now=now + 60) # minute suivante + + +def test_flood_global(store): + now = 1_800_000_000 + ok = sum(store.record(ev(u=f"{i:032x}"), UA, now=now) for i in range(200)) + assert ok == c.GLOBAL_PER_SEC + + +def test_aucune_ip_ni_ua_ni_uid_en_base(store, tmp_path): + store.record(ev(r="www.google.com"), UA) + store.db.execute("PRAGMA wal_checkpoint(FULL)") + cols = [r[1] for r in store.db.execute("PRAGMA table_info(hits)")] + assert cols == ["day", "vid", "path", "src", "dev", "nav"] + raw = b"".join(open(os.path.join(tmp_path, f), "rb").read() for f in os.listdir(tmp_path)) + for secret in (UID.encode(), b"127.0.0.1", b"Gecko/20100101", b"google.com"): + assert secret not in raw + assert store.db.execute("SELECT src, nav FROM hits").fetchone() == ("recherche", "firefox") + + +def test_sel_mensuel_non_reliable(store): + jan, fev = time.mktime((2027, 1, 15, 12, 0, 0, 0, 0, 0)), time.mktime((2027, 2, 15, 12, 0, 0, 0, 0, 0)) + store.record(ev(), UA, now=jan) + store.record(ev(), UA, now=fev) + vids = [r[0] for r in store.db.execute("SELECT vid FROM hits")] + assert len(set(vids)) == 2 + + +def test_agregation_et_purge(store): + now = time.time() + for i, uid in enumerate([UID, UID, "f" * 32]): + store.record(ev(u=uid, p=f"/p{i}"), UA, now=now) + store.db.execute("INSERT INTO hits VALUES('2000-01-01','x','/','direct','mobile','autre')") + a.purge(store.db, now) + out = a.aggregate(store.db) + day = time.strftime("%Y-%m-%d", time.gmtime(now)) + assert out["jours"] == {day: {"visiteurs": 2, "pages_vues": 3}} + assert out["sources"] == {"direct": 3} + + +@pytest.mark.parametrize("h,s", [("", "direct"), ("gastinel.me", "interne"), ("mind.gastinel.me", "interne"), + ("www.qwant.com", "recherche"), ("www.youtube.com", "social"), ("exemple.org", "autre"), + ("notgastinel.me", "autre")]) +def test_sources(h, s): + assert c.source_of(h) == s + + +def test_http_bout_en_bout(tmp_path): + st = c.Store(c.open_db(str(tmp_path / "h.db"))) + srv = ThreadingHTTPServer(("127.0.0.1", 0), c.make_handler(st, {"https://gastinel.me"})) + threading.Thread(target=srv.serve_forever, daemon=True).start() + url = f"http://127.0.0.1:{srv.server_address[1]}/m" + + def post(data, origin="https://gastinel.me"): + r = urllib.request.Request(url, data=data, headers={"Origin": origin, "User-Agent": UA, "Content-Type": "text/plain"}) + try: + return urllib.request.urlopen(r).status + except urllib.error.HTTPError as e: + return e.code + + assert post(ev()) == 204 + assert post(ev(), origin="https://evil.example") == 204 # réponse identique, mais rien d'enregistré + assert post(b"x" * 600) == 413 + srv.shutdown() + assert st.db.execute("SELECT COUNT(*) FROM hits").fetchone()[0] == 1