Prototype mesure d'audience par UID localStorage (collecteur, snippet, agrégateur, tests)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Cette révision appartient à :
Théo GastineletClaude Sonnet 5.5 a révisé 2026-10-24 15:21:38 +02:00
révision 41c57f6cd2
7 fichiers modifiés
+414

Aucun fichier ne correspond à votre recherche

+4
Voir le fichier
@@ -0,0 +1,4 @@
__pycache__/
*.sqlite3*
*.json
.pytest_cache/
+49
Voir le fichier
@@ -0,0 +1,49 @@
"""Agrégation quotidienne : SQLite -> bloc JSON pour data.json (stats.gastinel.me).
Purge : lignes brutes > RAW_DAYS jours, sels des mois révolus (les identifiants
des mois passés deviennent alors définitivement non reliables).
Les agrégats (des compteurs, sans identifiant) peuvent être gardés 25 mois.
"""
import argparse, json, sqlite3, time
from collector import open_db
RAW_DAYS = 62 # couvre le mois en cours + le précédent (visiteurs uniques mensuels)
def purge(db, now=None):
now = time.time() if now is None else now
cutoff = time.strftime("%Y-%m-%d", time.gmtime(now - RAW_DAYS * 86400))
db.execute("DELETE FROM hits WHERE day < ?", (cutoff,))
t = time.gmtime(now)
prev = f"{t.tm_year - (t.tm_mon == 1)}-{(t.tm_mon - 2) % 12 + 1:02d}"
db.execute("DELETE FROM salts WHERE month < ?", (prev,))
def aggregate(db):
q = lambda sql: db.execute(sql).fetchall()
return {
"methode": "uid-localstorage-v1",
"jours": {d: {"visiteurs": v, "pages_vues": p} for d, v, p in
q("SELECT day, COUNT(DISTINCT vid), COUNT(*) FROM hits GROUP BY day ORDER BY day")},
"mois": {m: {"visiteurs": v, "pages_vues": p} for m, v, p in
q("SELECT substr(day,1,7) m, COUNT(DISTINCT vid), COUNT(*) FROM hits GROUP BY m ORDER BY m")},
"sources": dict(q("SELECT src, COUNT(*) FROM hits GROUP BY src ORDER BY 2 DESC")),
"pages": dict(q("SELECT path, COUNT(*) FROM hits GROUP BY path ORDER BY 2 DESC LIMIT 20")),
"appareils": dict(q("SELECT dev, COUNT(DISTINCT vid) FROM hits GROUP BY dev")),
"navigateurs": dict(q("SELECT nav, COUNT(DISTINCT vid) FROM hits GROUP BY nav")),
}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--db", default="audience.sqlite3")
ap.add_argument("--out", default="audience.json")
a = ap.parse_args()
db = open_db(a.db)
purge(db)
with open(a.out, "w", encoding="utf-8") as f:
json.dump(aggregate(db), f, ensure_ascii=False, indent=1)
if __name__ == "__main__":
main()
+193
Voir le fichier
@@ -0,0 +1,193 @@
"""Collecteur de mesure d'audience (stdlib uniquement).
POST /m -> 204, corps JSON <= 512 o envoyé par navigator.sendBeacon (text/plain).
Aucune IP, aucun User-Agent complet n'est stocké : seulement famille de navigateur
grossière et type d'appareil. L'UID est pseudonymisé par HMAC avec un sel mensuel
(on ne peut pas relier un appareil d'un mois sur l'autre, ni retrouver l'UID d'origine).
"""
import argparse, hashlib, hmac, json, os, re, secrets, sqlite3, threading, time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import urlsplit
MAX_BODY = 512
UID_RE = re.compile(r"^[0-9a-f]{32}$")
PATH_RE = re.compile(r"^/[A-Za-z0-9._~/%-]{0,199}$")
HOST_RE = re.compile(r"^[a-z0-9.-]{0,100}$")
BOT_RE = re.compile(r"bot|crawl|spider|slurp|headless|phantom|python|curl|wget|httpclient|java/|go-http|scrapy|lighthouse|preview", re.I)
PER_UID_PER_MIN = 30
GLOBAL_PER_SEC = 50
OWN_HOSTS = ("gastinel.me",)
SEARCH = ("google.", "bing.", "duckduckgo.", "qwant.", "ecosia.", "yahoo.", "startpage.", "brave.")
SOCIAL = ("youtube.", "youtu.be", "instagram.", "facebook.", "t.co", "twitter.", "x.com", "tiktok.", "linkedin.", "reddit.", "discord", "whatsapp", "snapchat")
SCHEMA = """
CREATE TABLE IF NOT EXISTS hits(
day TEXT NOT NULL, -- AAAA-MM-JJ (UTC), pas d'heure précise
vid TEXT NOT NULL, -- HMAC(uid, sel du mois), 16 hex
path TEXT NOT NULL,
src TEXT NOT NULL, -- direct|interne|recherche|social|autre
dev TEXT NOT NULL, -- mobile|ordinateur
nav TEXT NOT NULL -- firefox|chrome|safari|edge|autre
);
CREATE INDEX IF NOT EXISTS hits_day ON hits(day);
CREATE TABLE IF NOT EXISTS salts(month TEXT PRIMARY KEY, salt BLOB NOT NULL);
"""
def open_db(path):
db = sqlite3.connect(path, check_same_thread=False, isolation_level=None)
db.execute("PRAGMA journal_mode=WAL")
db.executescript(SCHEMA)
return db
def source_of(ref_host):
h = (ref_host or "").lower()
if not h:
return "direct"
if any(h == o or h.endswith("." + o) for o in OWN_HOSTS):
return "interne"
if any(s in h for s in SEARCH):
return "recherche"
if any(s in h for s in SOCIAL):
return "social"
return "autre"
def browser_of(ua):
u = ua or ""
if "Firefox/" in u: return "firefox"
if "Edg/" in u: return "edge"
if "Chrome/" in u or "CriOS" in u: return "chrome"
if "Safari/" in u: return "safari"
return "autre"
class Limiter:
def __init__(self):
self.lock = threading.Lock()
self.per_uid = {} # uid -> (minute, count)
self.sec = (0, 0)
def allow(self, uid, now=None):
now = time.time() if now is None else now
m, s = int(now // 60), int(now)
with self.lock:
if self.sec[0] != s:
self.sec = (s, 0)
if self.sec[1] >= GLOBAL_PER_SEC:
return False
mm, c = self.per_uid.get(uid, (m, 0))
if mm != m:
c = 0
if len(self.per_uid) > 50000:
self.per_uid.clear()
if c >= PER_UID_PER_MIN:
return False
self.per_uid[uid] = (m, c + 1)
self.sec = (s, self.sec[1] + 1)
return True
class Store:
def __init__(self, db):
self.db, self.lock, self.limiter = db, threading.Lock(), Limiter()
def salt(self, month):
row = self.db.execute("SELECT salt FROM salts WHERE month=?", (month,)).fetchone()
if row:
return row[0]
s = secrets.token_bytes(32)
self.db.execute("INSERT OR IGNORE INTO salts VALUES(?,?)", (month, s))
return self.db.execute("SELECT salt FROM salts WHERE month=?", (month,)).fetchone()[0]
def record(self, raw, ua="", origin_ok=True, now=None):
"""Retourne True si l'événement est enregistré. Ne lève jamais."""
if not origin_ok or len(raw) > MAX_BODY or BOT_RE.search(ua or "") or not ua:
return False
try:
ev = json.loads(raw.decode("utf-8"))
except Exception:
return False
if not isinstance(ev, dict) or set(ev) - {"u", "p", "r", "m"}:
return False
uid, path, ref, mob = ev.get("u"), ev.get("p"), ev.get("r", ""), ev.get("m", 0)
if not (isinstance(uid, str) and UID_RE.match(uid)):
return False
if not (isinstance(path, str) and PATH_RE.match(path)):
return False
if not (isinstance(ref, str) and HOST_RE.match(ref)) or mob not in (0, 1):
return False
if not self.limiter.allow(uid, now):
return False
t = time.gmtime(time.time() if now is None else now)
day, month = time.strftime("%Y-%m-%d", t), time.strftime("%Y-%m", t)
with self.lock:
vid = hmac.new(self.salt(month), uid.encode(), hashlib.sha256).hexdigest()[:16]
self.db.execute("INSERT INTO hits VALUES(?,?,?,?,?,?)",
(day, vid, path, source_of(ref), "mobile" if mob else "ordinateur", browser_of(ua)))
return True
def make_handler(store, allowed_origins, demo_dir=None):
class H(BaseHTTPRequestHandler):
server_version = "m"
sys_version = ""
def log_message(self, *a): # pas de journal d'accès (il contiendrait l'IP)
pass
def _send(self, code, body=b"", ctype="text/plain"):
self.send_response(code)
self.send_header("Content-Type", ctype)
self.send_header("Content-Length", str(len(body)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
if body:
self.wfile.write(body)
def do_POST(self):
if urlsplit(self.path).path != "/m":
return self._send(404)
try:
n = int(self.headers.get("Content-Length", "0"))
except ValueError:
return self._send(400)
if n <= 0 or n > MAX_BODY:
return self._send(413 if n > MAX_BODY else 400)
raw = self.rfile.read(n)
org = self.headers.get("Origin", "")
ok = org in allowed_origins and self.headers.get("Sec-Fetch-Site", "same-origin") == "same-origin"
store.record(raw, self.headers.get("User-Agent", ""), ok)
self._send(204) # toujours 204 : ne rien apprendre aux robots
def do_GET(self):
if demo_dir is None:
return self._send(404)
p = urlsplit(self.path).path
name = {"/": "index.html", "/index.html": "index.html", "/page2.html": "page2.html", "/m.js": "m.js"}.get(p)
if not name:
return self._send(404)
with open(os.path.join(demo_dir, name), "rb") as f:
data = f.read()
self._send(200, data, "application/javascript" if name.endswith(".js") else "text/html; charset=utf-8")
return H
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--host", default="127.0.0.1")
ap.add_argument("--port", type=int, default=8097)
ap.add_argument("--db", default="audience.sqlite3")
ap.add_argument("--origin", action="append", default=[])
ap.add_argument("--demo", action="store_true", help="sert aussi demo/ (test local uniquement)")
a = ap.parse_args()
origins = set(a.origin or ["https://gastinel.me"])
demo = os.path.join(os.path.dirname(os.path.abspath(__file__)), "demo") if a.demo else None
srv = ThreadingHTTPServer((a.host, a.port), make_handler(Store(open_db(a.db)), origins, demo))
print(f"collecteur sur http://{a.host}:{a.port} pid={os.getpid()}", flush=True)
srv.serve_forever()
if __name__ == "__main__":
main()
+19
Voir le fichier
@@ -0,0 +1,19 @@
<!doctype html>
<html lang="fr"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Démo mesure d'audience</title>
<script src="/m.js" defer></script>
<style>body{font:16px system-ui;max-width:40rem;margin:2rem auto;padding:0 1rem}button{margin:.2rem}</style>
</head><body>
<h1>Démo mesure d'audience (local)</h1>
<p>Chaque chargement envoie un événement à <code>/m</code>. <a href="/page2.html">Page 2</a></p>
<p>UID stocké : <code id="u">?</code></p>
<button id="off">Ne plus être compté</button> <button id="on">Être compté à nouveau</button>
<script>
document.addEventListener("DOMContentLoaded", function () {
var show = function () { document.getElementById("u").textContent = localStorage.getItem("gm_uid") || (localStorage.getItem("gm_optout") ? "(opposition enregistrée)" : "(aucun)"); };
document.getElementById("off").onclick = function () { gmOptOut(); show(); };
document.getElementById("on").onclick = function () { gmOptOut(false); show(); };
show();
});
</script>
</body></html>
+24
Voir le fichier
@@ -0,0 +1,24 @@
/* Mesure d'audience gastinel.me — UID aléatoire en localStorage, 13 mois max, sans prolongation. */
(function (w, n, d) {
var K = "gm_uid", O = "gm_optout", MAX = 395 * 864e5, ls;
try { ls = w.localStorage; } catch (e) { return; }
w.gmOptOut = function (on) {
try { if (on === false) ls.removeItem(O); else { ls.setItem(O, "1"); ls.removeItem(K); } } catch (e) {}
};
if (n.doNotTrack == "1" || w.doNotTrack == "1" || n.globalPrivacyControl === true) return;
try { if (ls.getItem(O) == "1") return; } catch (e) { return; }
var v, now = Date.now();
try { v = JSON.parse(ls.getItem(K) || "null"); } catch (e) { v = null; }
if (!v || !/^[0-9a-f]{32}$/.test(v.id) || !(now - v.c < MAX) || v.c > now + 864e5) {
var b = new Uint8Array(16), h = "";
w.crypto.getRandomValues(b);
for (var i = 0; i < 16; i++) h += (b[i] + 256).toString(16).slice(1);
v = { id: h, c: now };
try { ls.setItem(K, JSON.stringify(v)); } catch (e) { return; }
}
var r = "";
try { r = d.referrer ? new URL(d.referrer).hostname : ""; } catch (e) {}
var body = JSON.stringify({ u: v.id, p: location.pathname.slice(0, 200), r: r.slice(0, 100),
m: w.matchMedia && w.matchMedia("(pointer:coarse)").matches ? 1 : 0 });
if (n.sendBeacon) n.sendBeacon("/m", body);
})(window, navigator, document);
+3
Voir le fichier
@@ -0,0 +1,3 @@
<!doctype html>
<html lang="fr"><head><meta charset="utf-8"><title>Démo page 2</title><script src="/m.js" defer></script></head>
<body><h1>Page 2</h1><p><a href="/">Retour</a></p></body></html>
+122
Voir le fichier
@@ -0,0 +1,122 @@
import json, os, sys, threading, time, urllib.request, urllib.error
from http.server import ThreadingHTTPServer
import pytest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import collector as c
import aggregate as a
UA = "Mozilla/5.0 (X11; Linux x86_64; rv:130.0) Gecko/20100101 Firefox/130.0"
UID = "0123456789abcdef0123456789abcdef"
def ev(**k):
d = {"u": UID, "p": "/", "r": "", "m": 0}
d.update(k)
return json.dumps(d).encode()
@pytest.fixture
def store(tmp_path):
return c.Store(c.open_db(str(tmp_path / "t.db")))
def test_uid_valide(store):
assert store.record(ev(), UA)
@pytest.mark.parametrize("uid", ["", "xyz", UID.upper(), UID + "0", UID[:-1], 123, None, "g" * 32])
def test_uid_invalide(store, uid):
assert not store.record(ev(u=uid), UA)
@pytest.mark.parametrize("bad", [{"p": "/a?q=email@x.fr"}, {"p": "http://x"}, {"p": "/" + "a" * 300},
{"r": "Evil Host"}, {"m": 2}, {"x": 1}])
def test_champs_refuses(store, bad):
assert not store.record(ev(**bad), UA)
def test_taille_max(store):
assert not store.record(b"{" + b" " * c.MAX_BODY + b"}", UA)
assert not store.record(b"pas du json", UA)
def test_bots_et_ua_vide(store):
assert not store.record(ev(), "Googlebot/2.1")
assert not store.record(ev(), "HeadlessChrome/120")
assert not store.record(ev(), "")
def test_origine_refusee(store):
assert not store.record(ev(), UA, origin_ok=False)
def test_flood_par_uid(store):
now = 1_800_000_000
ok = sum(store.record(ev(), UA, now=now) for _ in range(100))
assert ok == c.PER_UID_PER_MIN
assert store.record(ev(), UA, now=now + 60) # minute suivante
def test_flood_global(store):
now = 1_800_000_000
ok = sum(store.record(ev(u=f"{i:032x}"), UA, now=now) for i in range(200))
assert ok == c.GLOBAL_PER_SEC
def test_aucune_ip_ni_ua_ni_uid_en_base(store, tmp_path):
store.record(ev(r="www.google.com"), UA)
store.db.execute("PRAGMA wal_checkpoint(FULL)")
cols = [r[1] for r in store.db.execute("PRAGMA table_info(hits)")]
assert cols == ["day", "vid", "path", "src", "dev", "nav"]
raw = b"".join(open(os.path.join(tmp_path, f), "rb").read() for f in os.listdir(tmp_path))
for secret in (UID.encode(), b"127.0.0.1", b"Gecko/20100101", b"google.com"):
assert secret not in raw
assert store.db.execute("SELECT src, nav FROM hits").fetchone() == ("recherche", "firefox")
def test_sel_mensuel_non_reliable(store):
jan, fev = time.mktime((2027, 1, 15, 12, 0, 0, 0, 0, 0)), time.mktime((2027, 2, 15, 12, 0, 0, 0, 0, 0))
store.record(ev(), UA, now=jan)
store.record(ev(), UA, now=fev)
vids = [r[0] for r in store.db.execute("SELECT vid FROM hits")]
assert len(set(vids)) == 2
def test_agregation_et_purge(store):
now = time.time()
for i, uid in enumerate([UID, UID, "f" * 32]):
store.record(ev(u=uid, p=f"/p{i}"), UA, now=now)
store.db.execute("INSERT INTO hits VALUES('2000-01-01','x','/','direct','mobile','autre')")
a.purge(store.db, now)
out = a.aggregate(store.db)
day = time.strftime("%Y-%m-%d", time.gmtime(now))
assert out["jours"] == {day: {"visiteurs": 2, "pages_vues": 3}}
assert out["sources"] == {"direct": 3}
@pytest.mark.parametrize("h,s", [("", "direct"), ("gastinel.me", "interne"), ("mind.gastinel.me", "interne"),
("www.qwant.com", "recherche"), ("www.youtube.com", "social"), ("exemple.org", "autre"),
("notgastinel.me", "autre")])
def test_sources(h, s):
assert c.source_of(h) == s
def test_http_bout_en_bout(tmp_path):
st = c.Store(c.open_db(str(tmp_path / "h.db")))
srv = ThreadingHTTPServer(("127.0.0.1", 0), c.make_handler(st, {"https://gastinel.me"}))
threading.Thread(target=srv.serve_forever, daemon=True).start()
url = f"http://127.0.0.1:{srv.server_address[1]}/m"
def post(data, origin="https://gastinel.me"):
r = urllib.request.Request(url, data=data, headers={"Origin": origin, "User-Agent": UA, "Content-Type": "text/plain"})
try:
return urllib.request.urlopen(r).status
except urllib.error.HTTPError as e:
return e.code
assert post(ev()) == 204
assert post(ev(), origin="https://evil.example") == 204 # réponse identique, mais rien d'enregistré
assert post(b"x" * 600) == 413
srv.shutdown()
assert st.db.execute("SELECT COUNT(*) FROM hits").fetchone()[0] == 1