Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
123 lignes
4.4 KiB
Python
123 lignes
4.4 KiB
Python
import json, os, sys, threading, time, urllib.request, urllib.error
|
|
from http.server import ThreadingHTTPServer
|
|
import pytest
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
import collector as c
|
|
import aggregate as a
|
|
|
|
UA = "Mozilla/5.0 (X11; Linux x86_64; rv:130.0) Gecko/20100101 Firefox/130.0"
|
|
UID = "0123456789abcdef0123456789abcdef"
|
|
|
|
|
|
def ev(**k):
|
|
d = {"u": UID, "p": "/", "r": "", "m": 0}
|
|
d.update(k)
|
|
return json.dumps(d).encode()
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path):
|
|
return c.Store(c.open_db(str(tmp_path / "t.db")))
|
|
|
|
|
|
def test_uid_valide(store):
|
|
assert store.record(ev(), UA)
|
|
|
|
|
|
@pytest.mark.parametrize("uid", ["", "xyz", UID.upper(), UID + "0", UID[:-1], 123, None, "g" * 32])
|
|
def test_uid_invalide(store, uid):
|
|
assert not store.record(ev(u=uid), UA)
|
|
|
|
|
|
@pytest.mark.parametrize("bad", [{"p": "/a?q=email@x.fr"}, {"p": "http://x"}, {"p": "/" + "a" * 300},
|
|
{"r": "Evil Host"}, {"m": 2}, {"x": 1}])
|
|
def test_champs_refuses(store, bad):
|
|
assert not store.record(ev(**bad), UA)
|
|
|
|
|
|
def test_taille_max(store):
|
|
assert not store.record(b"{" + b" " * c.MAX_BODY + b"}", UA)
|
|
assert not store.record(b"pas du json", UA)
|
|
|
|
|
|
def test_bots_et_ua_vide(store):
|
|
assert not store.record(ev(), "Googlebot/2.1")
|
|
assert not store.record(ev(), "HeadlessChrome/120")
|
|
assert not store.record(ev(), "")
|
|
|
|
|
|
def test_origine_refusee(store):
|
|
assert not store.record(ev(), UA, origin_ok=False)
|
|
|
|
|
|
def test_flood_par_uid(store):
|
|
now = 1_800_000_000
|
|
ok = sum(store.record(ev(), UA, now=now) for _ in range(100))
|
|
assert ok == c.PER_UID_PER_MIN
|
|
assert store.record(ev(), UA, now=now + 60) # minute suivante
|
|
|
|
|
|
def test_flood_global(store):
|
|
now = 1_800_000_000
|
|
ok = sum(store.record(ev(u=f"{i:032x}"), UA, now=now) for i in range(200))
|
|
assert ok == c.GLOBAL_PER_SEC
|
|
|
|
|
|
def test_aucune_ip_ni_ua_ni_uid_en_base(store, tmp_path):
|
|
store.record(ev(r="www.google.com"), UA)
|
|
store.db.execute("PRAGMA wal_checkpoint(FULL)")
|
|
cols = [r[1] for r in store.db.execute("PRAGMA table_info(hits)")]
|
|
assert cols == ["day", "vid", "path", "src", "dev", "nav"]
|
|
raw = b"".join(open(os.path.join(tmp_path, f), "rb").read() for f in os.listdir(tmp_path))
|
|
for secret in (UID.encode(), b"127.0.0.1", b"Gecko/20100101", b"google.com"):
|
|
assert secret not in raw
|
|
assert store.db.execute("SELECT src, nav FROM hits").fetchone() == ("recherche", "firefox")
|
|
|
|
|
|
def test_sel_mensuel_non_reliable(store):
|
|
jan, fev = time.mktime((2027, 1, 15, 12, 0, 0, 0, 0, 0)), time.mktime((2027, 2, 15, 12, 0, 0, 0, 0, 0))
|
|
store.record(ev(), UA, now=jan)
|
|
store.record(ev(), UA, now=fev)
|
|
vids = [r[0] for r in store.db.execute("SELECT vid FROM hits")]
|
|
assert len(set(vids)) == 2
|
|
|
|
|
|
def test_agregation_et_purge(store):
|
|
now = time.time()
|
|
for i, uid in enumerate([UID, UID, "f" * 32]):
|
|
store.record(ev(u=uid, p=f"/p{i}"), UA, now=now)
|
|
store.db.execute("INSERT INTO hits VALUES('2000-01-01','x','/','direct','mobile','autre')")
|
|
a.purge(store.db, now)
|
|
out = a.aggregate(store.db)
|
|
day = time.strftime("%Y-%m-%d", time.gmtime(now))
|
|
assert out["jours"] == {day: {"visiteurs": 2, "pages_vues": 3}}
|
|
assert out["sources"] == {"direct": 3}
|
|
|
|
|
|
@pytest.mark.parametrize("h,s", [("", "direct"), ("gastinel.me", "interne"), ("mind.gastinel.me", "interne"),
|
|
("www.qwant.com", "recherche"), ("www.youtube.com", "social"), ("exemple.org", "autre"),
|
|
("notgastinel.me", "autre")])
|
|
def test_sources(h, s):
|
|
assert c.source_of(h) == s
|
|
|
|
|
|
def test_http_bout_en_bout(tmp_path):
|
|
st = c.Store(c.open_db(str(tmp_path / "h.db")))
|
|
srv = ThreadingHTTPServer(("127.0.0.1", 0), c.make_handler(st, {"https://gastinel.me"}))
|
|
threading.Thread(target=srv.serve_forever, daemon=True).start()
|
|
url = f"http://127.0.0.1:{srv.server_address[1]}/m"
|
|
|
|
def post(data, origin="https://gastinel.me"):
|
|
r = urllib.request.Request(url, data=data, headers={"Origin": origin, "User-Agent": UA, "Content-Type": "text/plain"})
|
|
try:
|
|
return urllib.request.urlopen(r).status
|
|
except urllib.error.HTTPError as e:
|
|
return e.code
|
|
|
|
assert post(ev()) == 204
|
|
assert post(ev(), origin="https://evil.example") == 204 # réponse identique, mais rien d'enregistré
|
|
assert post(b"x" * 600) == 413
|
|
srv.shutdown()
|
|
assert st.db.execute("SELECT COUNT(*) FROM hits").fetchone()[0] == 1
|