Prototype mesure d'audience par UID localStorage (collecteur, snippet, agrégateur, tests)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Cette révision appartient à :
révision
41c57f6cd2
7 fichiers modifiés
+414
Aucun fichier ne correspond à votre recherche
@@ -0,0 +1,193 @@
|
||||
"""Collecteur de mesure d'audience (stdlib uniquement).
|
||||
|
||||
POST /m -> 204, corps JSON <= 512 o envoyé par navigator.sendBeacon (text/plain).
|
||||
Aucune IP, aucun User-Agent complet n'est stocké : seulement famille de navigateur
|
||||
grossière et type d'appareil. L'UID est pseudonymisé par HMAC avec un sel mensuel
|
||||
(on ne peut pas relier un appareil d'un mois sur l'autre, ni retrouver l'UID d'origine).
|
||||
"""
|
||||
import argparse, hashlib, hmac, json, os, re, secrets, sqlite3, threading, time
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
MAX_BODY = 512
|
||||
UID_RE = re.compile(r"^[0-9a-f]{32}$")
|
||||
PATH_RE = re.compile(r"^/[A-Za-z0-9._~/%-]{0,199}$")
|
||||
HOST_RE = re.compile(r"^[a-z0-9.-]{0,100}$")
|
||||
BOT_RE = re.compile(r"bot|crawl|spider|slurp|headless|phantom|python|curl|wget|httpclient|java/|go-http|scrapy|lighthouse|preview", re.I)
|
||||
PER_UID_PER_MIN = 30
|
||||
GLOBAL_PER_SEC = 50
|
||||
OWN_HOSTS = ("gastinel.me",)
|
||||
SEARCH = ("google.", "bing.", "duckduckgo.", "qwant.", "ecosia.", "yahoo.", "startpage.", "brave.")
|
||||
SOCIAL = ("youtube.", "youtu.be", "instagram.", "facebook.", "t.co", "twitter.", "x.com", "tiktok.", "linkedin.", "reddit.", "discord", "whatsapp", "snapchat")
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS hits(
|
||||
day TEXT NOT NULL, -- AAAA-MM-JJ (UTC), pas d'heure précise
|
||||
vid TEXT NOT NULL, -- HMAC(uid, sel du mois), 16 hex
|
||||
path TEXT NOT NULL,
|
||||
src TEXT NOT NULL, -- direct|interne|recherche|social|autre
|
||||
dev TEXT NOT NULL, -- mobile|ordinateur
|
||||
nav TEXT NOT NULL -- firefox|chrome|safari|edge|autre
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS hits_day ON hits(day);
|
||||
CREATE TABLE IF NOT EXISTS salts(month TEXT PRIMARY KEY, salt BLOB NOT NULL);
|
||||
"""
|
||||
|
||||
|
||||
def open_db(path):
|
||||
db = sqlite3.connect(path, check_same_thread=False, isolation_level=None)
|
||||
db.execute("PRAGMA journal_mode=WAL")
|
||||
db.executescript(SCHEMA)
|
||||
return db
|
||||
|
||||
|
||||
def source_of(ref_host):
|
||||
h = (ref_host or "").lower()
|
||||
if not h:
|
||||
return "direct"
|
||||
if any(h == o or h.endswith("." + o) for o in OWN_HOSTS):
|
||||
return "interne"
|
||||
if any(s in h for s in SEARCH):
|
||||
return "recherche"
|
||||
if any(s in h for s in SOCIAL):
|
||||
return "social"
|
||||
return "autre"
|
||||
|
||||
|
||||
def browser_of(ua):
|
||||
u = ua or ""
|
||||
if "Firefox/" in u: return "firefox"
|
||||
if "Edg/" in u: return "edge"
|
||||
if "Chrome/" in u or "CriOS" in u: return "chrome"
|
||||
if "Safari/" in u: return "safari"
|
||||
return "autre"
|
||||
|
||||
|
||||
class Limiter:
|
||||
def __init__(self):
|
||||
self.lock = threading.Lock()
|
||||
self.per_uid = {} # uid -> (minute, count)
|
||||
self.sec = (0, 0)
|
||||
|
||||
def allow(self, uid, now=None):
|
||||
now = time.time() if now is None else now
|
||||
m, s = int(now // 60), int(now)
|
||||
with self.lock:
|
||||
if self.sec[0] != s:
|
||||
self.sec = (s, 0)
|
||||
if self.sec[1] >= GLOBAL_PER_SEC:
|
||||
return False
|
||||
mm, c = self.per_uid.get(uid, (m, 0))
|
||||
if mm != m:
|
||||
c = 0
|
||||
if len(self.per_uid) > 50000:
|
||||
self.per_uid.clear()
|
||||
if c >= PER_UID_PER_MIN:
|
||||
return False
|
||||
self.per_uid[uid] = (m, c + 1)
|
||||
self.sec = (s, self.sec[1] + 1)
|
||||
return True
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, db):
|
||||
self.db, self.lock, self.limiter = db, threading.Lock(), Limiter()
|
||||
|
||||
def salt(self, month):
|
||||
row = self.db.execute("SELECT salt FROM salts WHERE month=?", (month,)).fetchone()
|
||||
if row:
|
||||
return row[0]
|
||||
s = secrets.token_bytes(32)
|
||||
self.db.execute("INSERT OR IGNORE INTO salts VALUES(?,?)", (month, s))
|
||||
return self.db.execute("SELECT salt FROM salts WHERE month=?", (month,)).fetchone()[0]
|
||||
|
||||
def record(self, raw, ua="", origin_ok=True, now=None):
|
||||
"""Retourne True si l'événement est enregistré. Ne lève jamais."""
|
||||
if not origin_ok or len(raw) > MAX_BODY or BOT_RE.search(ua or "") or not ua:
|
||||
return False
|
||||
try:
|
||||
ev = json.loads(raw.decode("utf-8"))
|
||||
except Exception:
|
||||
return False
|
||||
if not isinstance(ev, dict) or set(ev) - {"u", "p", "r", "m"}:
|
||||
return False
|
||||
uid, path, ref, mob = ev.get("u"), ev.get("p"), ev.get("r", ""), ev.get("m", 0)
|
||||
if not (isinstance(uid, str) and UID_RE.match(uid)):
|
||||
return False
|
||||
if not (isinstance(path, str) and PATH_RE.match(path)):
|
||||
return False
|
||||
if not (isinstance(ref, str) and HOST_RE.match(ref)) or mob not in (0, 1):
|
||||
return False
|
||||
if not self.limiter.allow(uid, now):
|
||||
return False
|
||||
t = time.gmtime(time.time() if now is None else now)
|
||||
day, month = time.strftime("%Y-%m-%d", t), time.strftime("%Y-%m", t)
|
||||
with self.lock:
|
||||
vid = hmac.new(self.salt(month), uid.encode(), hashlib.sha256).hexdigest()[:16]
|
||||
self.db.execute("INSERT INTO hits VALUES(?,?,?,?,?,?)",
|
||||
(day, vid, path, source_of(ref), "mobile" if mob else "ordinateur", browser_of(ua)))
|
||||
return True
|
||||
|
||||
|
||||
def make_handler(store, allowed_origins, demo_dir=None):
|
||||
class H(BaseHTTPRequestHandler):
|
||||
server_version = "m"
|
||||
sys_version = ""
|
||||
|
||||
def log_message(self, *a): # pas de journal d'accès (il contiendrait l'IP)
|
||||
pass
|
||||
|
||||
def _send(self, code, body=b"", ctype="text/plain"):
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", ctype)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
if body:
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_POST(self):
|
||||
if urlsplit(self.path).path != "/m":
|
||||
return self._send(404)
|
||||
try:
|
||||
n = int(self.headers.get("Content-Length", "0"))
|
||||
except ValueError:
|
||||
return self._send(400)
|
||||
if n <= 0 or n > MAX_BODY:
|
||||
return self._send(413 if n > MAX_BODY else 400)
|
||||
raw = self.rfile.read(n)
|
||||
org = self.headers.get("Origin", "")
|
||||
ok = org in allowed_origins and self.headers.get("Sec-Fetch-Site", "same-origin") == "same-origin"
|
||||
store.record(raw, self.headers.get("User-Agent", ""), ok)
|
||||
self._send(204) # toujours 204 : ne rien apprendre aux robots
|
||||
|
||||
def do_GET(self):
|
||||
if demo_dir is None:
|
||||
return self._send(404)
|
||||
p = urlsplit(self.path).path
|
||||
name = {"/": "index.html", "/index.html": "index.html", "/page2.html": "page2.html", "/m.js": "m.js"}.get(p)
|
||||
if not name:
|
||||
return self._send(404)
|
||||
with open(os.path.join(demo_dir, name), "rb") as f:
|
||||
data = f.read()
|
||||
self._send(200, data, "application/javascript" if name.endswith(".js") else "text/html; charset=utf-8")
|
||||
return H
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--host", default="127.0.0.1")
|
||||
ap.add_argument("--port", type=int, default=8097)
|
||||
ap.add_argument("--db", default="audience.sqlite3")
|
||||
ap.add_argument("--origin", action="append", default=[])
|
||||
ap.add_argument("--demo", action="store_true", help="sert aussi demo/ (test local uniquement)")
|
||||
a = ap.parse_args()
|
||||
origins = set(a.origin or ["https://gastinel.me"])
|
||||
demo = os.path.join(os.path.dirname(os.path.abspath(__file__)), "demo") if a.demo else None
|
||||
srv = ThreadingHTTPServer((a.host, a.port), make_handler(Store(open_db(a.db)), origins, demo))
|
||||
print(f"collecteur sur http://{a.host}:{a.port} pid={os.getpid()}", flush=True)
|
||||
srv.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Référencer dans un nouveau ticket
Bloquer un utilisateur