Fichiers

199 lignes
7.8 KiB
Python

"""Collecteur de mesure d'audience (stdlib uniquement).
POST /m -> 204, corps JSON <= 512 o envoyé par navigator.sendBeacon (text/plain).
Aucune IP, aucun User-Agent complet n'est stocké : seulement famille de navigateur
grossière et type d'appareil. L'UID est pseudonymisé par HMAC avec un sel mensuel
(on ne peut pas relier un appareil d'un mois sur l'autre, ni retrouver l'UID d'origine).
"""
import argparse, hashlib, hmac, json, os, re, secrets, sqlite3, threading, time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import urlsplit
MAX_BODY = 512
UID_RE = re.compile(r"^[0-9a-f]{32}$")
PATH_RE = re.compile(r"^/[A-Za-z0-9._~/%-]{0,199}$")
HOST_RE = re.compile(r"^[a-z0-9.-]{0,100}$")
BOT_RE = re.compile(r"bot|crawl|spider|slurp|headless|phantom|python|curl|wget|httpclient|java/|go-http|scrapy|lighthouse|preview", re.I)
PER_UID_PER_MIN = 30
GLOBAL_PER_SEC = 50
OWN_HOSTS = ("gastinel.me",)
SEARCH = ("google.", "bing.", "duckduckgo.", "qwant.", "ecosia.", "yahoo.", "startpage.", "brave.")
SOCIAL = ("youtube.", "youtu.be", "instagram.", "facebook.", "t.co", "twitter.", "x.com", "tiktok.", "linkedin.", "reddit.", "discord", "whatsapp", "snapchat")
SCHEMA = """
CREATE TABLE IF NOT EXISTS hits(
day TEXT NOT NULL, -- AAAA-MM-JJ (UTC), pas d'heure précise
vid TEXT NOT NULL, -- HMAC(uid, sel du mois), 16 hex
path TEXT NOT NULL,
src TEXT NOT NULL, -- direct|interne|recherche|social|autre
dev TEXT NOT NULL, -- mobile|ordinateur
nav TEXT NOT NULL -- firefox|chrome|safari|edge|autre
);
CREATE INDEX IF NOT EXISTS hits_day ON hits(day);
CREATE TABLE IF NOT EXISTS salts(month TEXT PRIMARY KEY, salt BLOB NOT NULL);
"""
def open_db(path):
db = sqlite3.connect(path, check_same_thread=False, isolation_level=None)
db.execute("PRAGMA journal_mode=WAL")
db.executescript(SCHEMA)
return db
def source_of(ref_host):
h = (ref_host or "").lower()
if not h:
return "direct"
if any(h == o or h.endswith("." + o) for o in OWN_HOSTS):
return "interne"
if any(s in h for s in SEARCH):
return "recherche"
if any(s in h for s in SOCIAL):
return "social"
return "autre"
def browser_of(ua):
u = ua or ""
if "Firefox/" in u: return "firefox"
if "Edg/" in u: return "edge"
if "Chrome/" in u or "CriOS" in u: return "chrome"
if "Safari/" in u: return "safari"
return "autre"
class Limiter:
def __init__(self):
self.lock = threading.Lock()
self.per_uid = {} # uid -> (minute, count)
self.sec = (0, 0)
def allow(self, uid, now=None):
now = time.time() if now is None else now
m, s = int(now // 60), int(now)
with self.lock:
if self.sec[0] != s:
self.sec = (s, 0)
if self.sec[1] >= GLOBAL_PER_SEC:
return False
if uid not in self.per_uid and len(self.per_uid) >= 50000:
self.per_uid.clear()
mm, c = self.per_uid.get(uid, (m, 0))
if mm != m:
c = 0
if c >= PER_UID_PER_MIN:
return False
self.per_uid[uid] = (m, c + 1)
self.sec = (s, self.sec[1] + 1)
return True
class Store:
def __init__(self, db):
self.db, self.lock, self.limiter = db, threading.Lock(), Limiter()
def salt(self, month):
row = self.db.execute("SELECT salt FROM salts WHERE month=?", (month,)).fetchone()
if row:
return row[0]
s = secrets.token_bytes(32)
self.db.execute("INSERT OR IGNORE INTO salts VALUES(?,?)", (month, s))
return self.db.execute("SELECT salt FROM salts WHERE month=?", (month,)).fetchone()[0]
def record(self, raw, ua="", origin_ok=True, now=None):
"""Retourne True si l'événement est enregistré. Ne lève jamais."""
if not origin_ok or len(raw) > MAX_BODY or BOT_RE.search(ua or "") or not ua:
return False
try:
ev = json.loads(raw.decode("utf-8"))
except Exception:
return False
if not isinstance(ev, dict) or set(ev) - {"u", "p", "r", "m"}:
return False
uid, path, ref, mob = ev.get("u"), ev.get("p"), ev.get("r", ""), ev.get("m", 0)
if not (isinstance(uid, str) and UID_RE.match(uid)):
return False
if not (isinstance(path, str) and PATH_RE.match(path)):
return False
if not (isinstance(ref, str) and HOST_RE.match(ref)) or mob not in (0, 1):
return False
if not self.limiter.allow(uid, now):
return False
t = time.gmtime(time.time() if now is None else now)
day, month = time.strftime("%Y-%m-%d", t), time.strftime("%Y-%m", t)
with self.lock:
vid = hmac.new(self.salt(month), uid.encode(), hashlib.sha256).hexdigest()[:16]
self.db.execute("INSERT INTO hits VALUES(?,?,?,?,?,?)",
(day, vid, path, source_of(ref), "mobile" if mob else "ordinateur", browser_of(ua)))
return True
def make_handler(store, allowed_origins, demo_dir=None):
class H(BaseHTTPRequestHandler):
server_version = "m"
sys_version = ""
timeout = 5 # coupe les connexions lentes (slowloris)
def log_message(self, *a): # pas de journal d'accès (il contiendrait l'IP)
pass
def _send(self, code, body=b"", ctype="text/plain"):
self.send_response(code)
self.send_header("Content-Type", ctype)
self.send_header("Content-Length", str(len(body)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
if body:
self.wfile.write(body)
def do_POST(self):
if urlsplit(self.path).path != "/m":
return self._send(404)
try:
n = int(self.headers.get("Content-Length", "0"))
except ValueError:
return self._send(400)
if n <= 0 or n > MAX_BODY:
return self._send(413 if n > MAX_BODY else 400)
raw = self.rfile.read(n)
org = self.headers.get("Origin", "")
ok = org in allowed_origins and self.headers.get("Sec-Fetch-Site", "same-origin") == "same-origin"
store.record(raw, self.headers.get("User-Agent", ""), ok)
self._send(204) # toujours 204 : ne rien apprendre aux robots
def do_GET(self):
if demo_dir is None:
return self._send(404)
p = urlsplit(self.path).path
name = {"/": "index.html", "/index.html": "index.html", "/page2.html": "page2.html", "/m.js": "m.js"}.get(p)
if not name:
return self._send(404)
with open(os.path.join(demo_dir, name), "rb") as f:
data = f.read()
self._send(200, data, "application/javascript" if name.endswith(".js") else "text/html; charset=utf-8")
return H
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--host", default="127.0.0.1")
ap.add_argument("--port", type=int, default=8097)
ap.add_argument("--db", default="audience.sqlite3")
ap.add_argument("--origin", action="append", default=[])
ap.add_argument("--demo", action="store_true", help="sert aussi demo/ (test local uniquement)")
a = ap.parse_args()
origins = set(a.origin or ["https://gastinel.me"])
demo = os.path.join(os.path.dirname(os.path.abspath(__file__)), "demo") if a.demo else None
class Serveur(ThreadingHTTPServer):
def handle_error(self, request, client_address): # le défaut imprime l'adresse IP du client
pass
srv = Serveur((a.host, a.port), make_handler(Store(open_db(a.db)), origins, demo))
print(f"collecteur sur http://{a.host}:{a.port} pid={os.getpid()}", flush=True)
srv.serve_forever()
if __name__ == "__main__":
main()