From c3ba63e78e00de6084c26a9cdd3648d37002cfb4 Mon Sep 17 00:00:00 2001 From: Theo Gastinel Date: Wed, 30 Sep 2026 07:39:19 +0200 Subject: [PATCH] Original en lecture seule depuis cornafsuisse:~/gws-collecte/collecte.py (2026-09-30) --- collecte.py | 586 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 586 insertions(+) create mode 100644 collecte.py diff --git a/collecte.py b/collecte.py new file mode 100644 index 0000000..bc9e503 --- /dev/null +++ b/collecte.py @@ -0,0 +1,586 @@ +#!/usr/bin/env python3 +"""Collecte automatisee de l'historique Infoclimat (stdlib uniquement). + +Usage : + python3 collecte.py # tourne jusqu'au plafond du jour ou la fin de la file + python3 collecte.py --dry-run # construit la file, affiche les 5 premieres fenetres, aucun appel API + python3 collecte.py --max-requetes N # s'arrete apres N appels HTTP (test) + python3 collecte.py --maj # mise a jour quotidienne des stations finies (derniere date + 1 -> hier) + python3 collecte.py --maj --dry-run # fenetres de mise a jour prevues, aucun appel API +Voir README.md. +""" +import argparse, datetime as dt, fcntl, gzip, json, os, re, shutil, socket, sys, time, zlib +import urllib.error, urllib.parse, urllib.request +from pathlib import Path +from zoneinfo import ZoneInfo + +BASE = Path(__file__).resolve().parent +DONNEES = BASE / "donnees" +TOKEN_F = BASE / "infoclimat.token" +STATIONS_F = BASE / "stations.json" +ETAT_F = BASE / "etat.json" +FENETRES_F = BASE / "fenetres.jsonl" +QUOTA_F = BASE / "quota.json" +STATUT_F = BASE / "statut.json" +LOG_F = BASE / "collecte.log" +STOP_F = BASE / "STOP_RAISON.txt" +LOCK_F = BASE / ".collecte.lock" + +PARIS = ZoneInfo("Europe/Paris") +PLAFOND = 4500 +PAUSE = 1.2 # s entre deux debuts de requete (>= 1,1 s) +TIMEOUT = 300 +UA = "GastinelWeatherStudio/0.4 (collecte historique)" +PLANCHER = dt.date(2004, 1, 1) +MAX_DONNEES = 8 * 1024**3 +MAX_DISQUE = 0.85 +LOG_MAX = 5 * 1024**2 +MAJ_MAX = 1200 # requetes max par jour pour --maj ; aussi reservees par le run de nuit + +TOKEN = "" +_last = [0.0] +statut = {"debut_run": None, "derniere_erreur": None, "octets_run": 0, "fenetres_ok_run": 0, + "fenetres_vides_run": 0, "requetes_run": 0, "etat": "demarrage"} + + +# ---------------------------------------------------------------- utilitaires +def masquer(s): + s = str(s) + if TOKEN: + s = s.replace(TOKEN, "***").replace(urllib.parse.quote(TOKEN), "***") + return re.sub(r"token=[^&\s\"']+", "token=***", s) + + +def log(msg): + ligne = f"{dt.datetime.now(PARIS):%Y-%m-%d %H:%M:%S} {masquer(msg)}" + print(ligne, flush=True) + try: + if LOG_F.exists() and LOG_F.stat().st_size > LOG_MAX: + LOG_F.replace(LOG_F.with_suffix(".log.1")) + with open(LOG_F, "a", encoding="utf-8") as f: + f.write(ligne + "\n") + except OSError: + pass + + +def ecrire_json(path, obj): + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(obj, ensure_ascii=False, indent=1), encoding="utf-8") + os.replace(tmp, path) + + +def lire_json(path, defaut): + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return defaut + + +def aujourdhui(): + return dt.datetime.now(PARIS).date() + + +def d2s(d): + return d.isoformat() + + +def s2d(s): + return dt.date.fromisoformat(s) + + +# ---------------------------------------------------------------- quota +def quota_lire(): + q = lire_json(QUOTA_F, {}) + j = d2s(aujourdhui()) + if q.get("date") != j: + q = {"date": j, "requetes": 0} + return q + + +def quota_incr(): + q = quota_lire() + q["requetes"] += 1 + ecrire_json(QUOTA_F, q) + return q["requetes"] + + +def quota_restant(): + return PLAFOND - quota_lire()["requetes"] + + +# ---------------------------------------------------------------- disque +def taille_donnees(): + t = 0 + for racine, _, fichiers in os.walk(DONNEES): + for f in fichiers: + try: + t += os.path.getsize(os.path.join(racine, f)) + except OSError: + pass + return t + + +def disque_ok(): + u = shutil.disk_usage("/") + frac = u.used / u.total + td = taille_donnees() + if td > MAX_DONNEES or frac > MAX_DISQUE: + return False, f"donnees={td / 1e9:.2f} Go, disque /={frac:.0%}" + return True, "" + + +# ---------------------------------------------------------------- file de travail +def init_etat(): + """Etat par station ; construit une fois, conserve ensuite (reprise).""" + etat = lire_json(ETAT_F, None) + if etat: + return etat + st = lire_json(STATIONS_F, []) + fin = d2s(aujourdhui() - dt.timedelta(days=1)) + ordre = sorted(st, key=lambda s: (not s.get("donnees_recentes"), not s.get("date_debut"), s["id"])) + stations = {} + for s in ordre: + pas = s.get("pas_minutes") + active = bool(s.get("donnees_recentes")) + if s.get("date_debut"): + mode, cur = "avant", s["date_debut"] + "-01" + taille = 365 if (pas and pas >= 30) else 180 + elif active: + mode, cur, taille = "arriere", fin, 365 if (pas and pas >= 30) else 180 + else: + mode, cur, taille = "avant", d2s(PLANCHER), 365 + stations[s["id"]] = {"mode": mode, "curseur": cur, "fin": fin, "taille": taille, + "active": active, "vides_consec": 0, "fini": False, "erreurs": 0, + "fenetres_ok": 0, "fenetres_vides": 0, "octets": 0} + etat = {"cree": dt.datetime.now(PARIS).isoformat(timespec="seconds"), + "ordre": [s["id"] for s in ordre], "stations": stations} + ecrire_json(ETAT_F, etat) + return etat + + +def prochaine_fenetre(e): + cur, fin, t = s2d(e["curseur"]), s2d(e["fin"]), e["taille"] + if e["mode"] == "avant": + if cur > fin: + return None + return cur, min(cur + dt.timedelta(days=t - 1), fin) + if cur < PLANCHER: + return None + return max(cur - dt.timedelta(days=t - 1), PLANCHER), cur + + +def avancer(e, a, b, vide): + if e["mode"] == "avant": + e["curseur"] = d2s(b + dt.timedelta(days=1)) + if s2d(e["curseur"]) > s2d(e["fin"]): + e["fini"] = True + else: + e["vides_consec"] = e["vides_consec"] + 1 if vide else 0 + e["curseur"] = d2s(a - dt.timedelta(days=1)) + if e["vides_consec"] >= 2 or a <= PLANCHER: + e["fini"] = True + e["erreurs"] = 0 + + +def noter(sid, a, b, resultat, **kw): + with open(FENETRES_F, "a", encoding="utf-8") as f: + f.write(json.dumps({"id": sid, "start": d2s(a), "end": d2s(b), "res": resultat, + "t": dt.datetime.now(PARIS).isoformat(timespec="seconds"), **kw}) + "\n") + + +# ---------------------------------------------------------------- appel API +class Arret(Exception): + pass + + +def appeler(sid, a, b): + """Retourne (code, info). code : ok | vide | trop | timeout | limite | erreur.""" + url = ("https://www.infoclimat.fr/opendata/?version=2&method=get&format=json&stations[]=" + + urllib.parse.quote(sid) + f"&start={a}&end={b}&token=" + urllib.parse.quote(TOKEN)) + attente = PAUSE - (time.time() - _last[0]) + if attente > 0: + time.sleep(attente) + _last[0] = time.time() + n = quota_incr() + statut["requetes_run"] += 1 + rep = DONNEES / sid + rep.mkdir(parents=True, exist_ok=True) + final = rep / f"{a}_{b}.json.gz" + part = rep / f"{a}_{b}.json.gz.part" + req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Encoding": "gzip"}) + t0 = time.time() + brut = 0 + try: + with urllib.request.urlopen(req, timeout=TIMEOUT) as r: + deja_gz = (r.headers.get("Content-Encoding") or "").lower() == "gzip" + with open(part, "wb") as fo: + sortie = fo if deja_gz else gzip.GzipFile(fileobj=fo, mode="wb", compresslevel=6) + while True: + if time.time() - t0 > TIMEOUT: + raise socket.timeout("duree totale > timeout") + x = r.read(1 << 16) + if not x: + break + brut += len(x) + sortie.write(x) + if not deja_gz: + sortie.close() + except urllib.error.HTTPError as e: + part.unlink(missing_ok=True) + corps = masquer(e.read(2000).decode("utf-8", "replace")) + if e.code in (429, 503): + return "limite", f"HTTP {e.code}" + return analyser_texte(corps) or ("erreur", f"HTTP {e.code}: {corps[:150]}") + except (socket.timeout, TimeoutError) as e: + part.unlink(missing_ok=True) + return "timeout", masquer(e) + except (urllib.error.URLError, OSError, zlib.error) as e: + part.unlink(missing_ok=True) + return "erreur", f"{type(e).__name__}: {masquer(e)[:150]}" + duree = time.time() - t0 + code, info = valider(part) + if code == "ok": + os.replace(part, final) + taille = final.stat().st_size + statut["octets_run"] += taille + return "ok", {"lignes": info, "gz": taille, "duree": round(duree, 1), "quota": n} + part.unlink(missing_ok=True) + return code, info + + +def analyser_texte(txt): + bas = txt.lower() + if "wrong ip" in bas: + return "fatal", "Wrong ip address" + if "no account found" in bas: + return "fatal", "No account found" + if "too many lines" in bas: + return "trop", "Too many lines" + if "too many requests" in bas: + return "limite", "Too many requests" + return None + + +def valider(part): + """Relit le gz en flux : JSON objet, status OK, pas d'erreur, compte les dh_utc.""" + try: + with gzip.open(part, "rb") as f: + tete = f.read(4096) + txt = tete.decode("utf-8", "replace") + a = analyser_texte(txt[:2000]) + if a: + return a + if not txt.lstrip().startswith("{"): + return "erreur", "reponse non JSON: " + masquer(txt[:150]) + m = re.search(r'"status"\s*:\s*"([^"]*)"', txt) + if not m or m.group(1).upper() != "OK": + err = re.search(r'"errors"\s*:\s*\[\s*"([^"]+)"', txt) + return "erreur", "status non OK: " + masquer(err.group(1) if err else txt[:150]) + n = tete.count(b'"dh_utc"') + reste = tete[-16:] + dernier = tete + while True: + x = f.read(1 << 20) + if not x: + break + n += (reste + x).count(b'"dh_utc"') + reste = x[-16:] + dernier = x + if not dernier.rstrip().endswith(b"}"): + return "erreur", "JSON tronque" + return ("ok", n) if n else ("vide", 0) + except (OSError, EOFError, zlib.error) as e: + return "erreur", f"gz invalide: {e}" + + +# ---------------------------------------------------------------- statut +def maj_statut(etat, courant=None): + st = etat["stations"] + finies = [s for s in etat["ordre"] if st[s]["fini"]] + q = quota_lire() + if "derniere_maj" not in statut: + statut["derniere_maj"] = lire_json(STATUT_F, {}).get("derniere_maj") + statut.update({ + "maj": dt.datetime.now(PARIS).isoformat(timespec="seconds"), + "stations_total": len(st), "stations_finies": len(finies), + "stations_restantes": len(st) - len(finies), "station_en_cours": courant, + "curseur_en_cours": st[courant]["curseur"] if courant else None, + "requetes_du_jour": q["requetes"], "plafond": PLAFOND, "jour_quota": q["date"], + "octets_total_collectes": sum(s["octets"] for s in st.values()), + "fenetres_ok_total": sum(s["fenetres_ok"] for s in st.values()), + "fenetres_vides_total": sum(s["fenetres_vides"] for s in st.values()), + }) + ecrire_json(STATUT_F, statut) + + +def stop(raison): + STOP_F.write_text(f"{dt.datetime.now(PARIS).isoformat(timespec='seconds')} {raison}\n" + "Collecte arretee. Verifier la cle / l'IP puis supprimer ce fichier pour relancer.\n", + encoding="utf-8") + log(f"ARRET D'URGENCE : {raison}") + statut["derniere_erreur"] = raison + statut["etat"] = "arret_urgence" + + +# ---------------------------------------------------------------- boucle +def a_mettre_a_jour(etat): + """Stations actives finies dont la derniere date collectee est < hier : (sid, debut, hier).""" + hier = aujourdhui() - dt.timedelta(days=1) + res = [] + for sid in etat["ordre"]: + e = etat["stations"][sid] + if not (e["fini"] and e["active"]): + continue + debut = s2d(e.get("derniere", e["fin"])) + dt.timedelta(days=1) + if debut <= hier: + res.append((sid, debut, hier)) + return res + + +def reserve_maj(etat): + """Requetes laissees par le run de nuit pour la maj de 08:00 du meme jour Paris.""" + return min(MAJ_MAX, len(a_mettre_a_jour(etat)) + 20) + + +def traiter(etat, max_req): + st = etat["stations"] + reserve = reserve_maj(etat) + log(f"Reserve pour la maj de 08:00 : {reserve} requetes (run de nuit plafonne a {PLAFOND - reserve})") + limites = 0 + nreq = 0 + for sid in etat["ordre"]: + e = st[sid] + while not e["fini"]: + if STOP_F.exists(): + raise Arret("STOP_RAISON.txt present") + if quota_restant() <= reserve: + log(f"Plafond {PLAFOND} (- reserve maj {reserve}) requetes atteint pour le {quota_lire()['date']} : arret propre.") + statut["etat"] = "plafond_atteint" + return + if max_req and nreq >= max_req: + log(f"--max-requetes {max_req} atteint : arret.") + statut["etat"] = "arret_max_requetes" + return + if nreq % 25 == 0: + ok, pourquoi = disque_ok() + while not ok: + statut["etat"] = "pause_disque" + maj_statut(etat, sid) + log(f"Pause disque ({pourquoi}) : attente 30 min (rapatriement).") + time.sleep(1800) + ok, pourquoi = disque_ok() + statut["etat"] = "en_cours" + fen = prochaine_fenetre(e) + if not fen: + e["fini"] = True + break + a, b = fen + code, info = appeler(sid, a, b) + nreq += 1 + if code == "ok": + limites = 0 + e["fenetres_ok"] += 1 + e["octets"] += info["gz"] + statut["fenetres_ok_run"] += 1 + noter(sid, a, b, "ok", **info) + log(f"{sid} {a}..{b} OK {info['lignes']} lignes {info['gz'] / 1e6:.2f} Mo gz " + f"{info['duree']} s (quota {info['quota']})") + avancer(e, a, b, False) + elif code == "vide": + limites = 0 + e["fenetres_vides"] += 1 + statut["fenetres_vides_run"] += 1 + noter(sid, a, b, "vide") + log(f"{sid} {a}..{b} vide") + avancer(e, a, b, True) + elif code in ("trop", "timeout"): + if e["taille"] > 1: + e["taille"] = max(1, e["taille"] // 2) + log(f"{sid} {a}..{b} {code} -> fenetre reduite a {e['taille']} j") + else: + e["erreurs"] += 1 + if e["erreurs"] >= 3: + noter(sid, a, b, "abandon", info=str(info)) + log(f"{sid} {a}..{b} {code} sur 1 jour x3 : fenetre abandonnee") + avancer(e, a, b, False) + elif code == "limite": + limites += 1 + statut["derniere_erreur"] = f"{sid} {a}..{b} {info}" + if limites >= 4: + log("Limitation serveur persistante : fin du run (reprise au prochain).") + statut["etat"] = "limitation_serveur" + return + log(f"{info} : pause 15 min") + maj_statut(etat, sid) + time.sleep(900) + elif code == "fatal": + stop(info) + raise Arret(info) + else: + e["erreurs"] += 1 + statut["derniere_erreur"] = f"{sid} {a}..{b} {info}" + log(f"{sid} {a}..{b} ERREUR {info} (essai {e['erreurs']})") + if e["erreurs"] >= 3: + noter(sid, a, b, "erreur", info=str(info)) + avancer(e, a, b, False) + if not e["fini"] and e["mode"] == "arriere": + pass + else: + time.sleep(10 * e["erreurs"]) + ecrire_json(ETAT_F, etat) + maj_statut(etat, sid) + ecrire_json(ETAT_F, etat) + statut["etat"] = "termine" + log("File de travail terminee.") + + +def traiter_maj(etat, max_req): + """Une requete par station finie : (derniere date + 1) .. hier. Quota partage, <= MAJ_MAX.""" + st = etat["stations"] + liste = a_mettre_a_jour(etat) + log(f"Maj : {len(liste)} stations a mettre a jour") + nreq = limites = 0 + for sid, a, b in liste: + e = st[sid] + while True: + if STOP_F.exists(): + raise Arret("STOP_RAISON.txt present") + if quota_restant() <= 0 or nreq >= min(MAJ_MAX, max_req or MAJ_MAX): + log(f"Maj : limite atteinte ({nreq} requetes maj, quota du jour " + f"{quota_lire()['requetes']}/{PLAFOND}) : reprise demain.") + statut["etat"] = "maj_plafond" + return + if nreq % 25 == 0: + ok, pourquoi = disque_ok() + if not ok: + log(f"Maj : disque plein ({pourquoi}) : arret, reprise demain.") + statut["etat"] = "maj_pause_disque" + return + code, info = appeler(sid, a, b) + nreq += 1 + if code == "limite": + limites += 1 + statut["derniere_erreur"] = f"maj {sid} {a}..{b} {info}" + if limites >= 4: + log("Maj : limitation serveur persistante : fin.") + statut["etat"] = "limitation_serveur" + return + log(f"{info} : pause 15 min") + time.sleep(900) + continue + if code == "fatal": + stop(info) + raise Arret(info) + limites = 0 + if code == "ok": + e["fenetres_ok"] += 1 + e["octets"] += info["gz"] + statut["fenetres_ok_run"] += 1 + noter(sid, a, b, "ok", maj=True, **info) + log(f"maj {sid} {a}..{b} OK {info['lignes']} lignes (quota {info['quota']})") + e["derniere"] = d2s(b) + elif code == "vide": + e["fenetres_vides"] += 1 + statut["fenetres_vides_run"] += 1 + noter(sid, a, b, "vide", maj=True) + log(f"maj {sid} {a}..{b} vide") + e["derniere"] = d2s(b) + else: # erreur / trop / timeout : on ne bouge pas, la fenetre sera retentee demain (elargie) + statut["derniere_erreur"] = f"maj {sid} {a}..{b} {info}" + noter(sid, a, b, "erreur", maj=True, info=str(info)) + log(f"maj {sid} {a}..{b} {code} {info} : reessai demain") + ecrire_json(ETAT_F, etat) + break + statut["etat"] = "maj_terminee" + log("Maj terminee.") + + +def dry_run_maj(etat): + liste = a_mettre_a_jour(etat) + q = quota_lire() + print(f"Maj : {len(liste)} stations finies a mettre a jour ; quota du jour ({q['date']}) " + f"{q['requetes']}/{PLAFOND} ; max {MAJ_MAX} requetes maj/jour ; reserve de nuit {reserve_maj(etat)}") + for sid, a, b in liste[:10]: + print(f" {sid} {a}..{b} ({(b - a).days + 1} j) -> donnees/{sid}/{a}_{b}.json.gz") + if len(liste) > 10: + print(f" ... +{len(liste) - 10} autres") + + +def dry_run(etat): + st = etat["stations"] + modes = {} + for s in st.values(): + k = f"{'active' if s['active'] else 'arretee'}/{s['mode']}/{s['taille']}j" + modes[k] = modes.get(k, 0) + 1 + print(f"Stations : {len(st)}, finies : {sum(s['fini'] for s in st.values())}") + for k, v in sorted(modes.items()): + print(f" {k:28s} {v}") + print(f"Quota du jour ({quota_lire()['date']}) : {quota_lire()['requetes']}/{PLAFOND}") + print(f"Disque : {disque_ok()}") + print("5 premieres fenetres (sans appel API) :") + n = 0 + for sid in etat["ordre"]: + e = dict(st[sid]) + while n < 5 and not e["fini"]: + f = prochaine_fenetre(e) + if not f: + break + print(f" {sid} {f[0]}..{f[1]} (mode {e['mode']}, {e['taille']} j)") + avancer(e, f[0], f[1], False) + n += 1 + if n >= 5: + break + + +def main(): + global TOKEN + ap = argparse.ArgumentParser() + ap.add_argument("--dry-run", action="store_true") + ap.add_argument("--max-requetes", type=int, default=0) + ap.add_argument("--maj", action="store_true") + args = ap.parse_args() + os.chdir(BASE) + DONNEES.mkdir(exist_ok=True) + etat = init_etat() + if args.dry_run: + return dry_run_maj(etat) if args.maj else dry_run(etat) + if STOP_F.exists(): + print(f"STOP_RAISON.txt present : rien n'est lance.\n{STOP_F.read_text()}") + return 2 + lock = open(LOCK_F, "w") + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + print("Une collecte tourne deja.") + return 0 + TOKEN = TOKEN_F.read_text(encoding="utf-8").strip() + for p in DONNEES.rglob("*.part"): + p.unlink(missing_ok=True) + statut["debut_run"] = dt.datetime.now(PARIS).isoformat(timespec="seconds") + statut["etat"] = "en_cours" + if args.maj: + statut["derniere_maj"] = statut["debut_run"] + log(f"Debut du {'run MAJ' if args.maj else 'run'} (quota du jour {quota_lire()['requetes']}/{PLAFOND})") + try: + (traiter_maj if args.maj else traiter)(etat, args.max_requetes) + except Arret as e: + log(f"Arret : {e}") + return 3 + except KeyboardInterrupt: + statut["etat"] = "interrompu" + log("Interrompu.") + except Exception as e: + statut["etat"] = "plante" + statut["derniere_erreur"] = masquer(f"{type(e).__name__}: {e}") + log(f"PLANTAGE {statut['derniere_erreur']}") + raise + finally: + ecrire_json(ETAT_F, etat) + maj_statut(etat) + log(f"Fin du run : {statut['requetes_run']} requetes, {statut['fenetres_ok_run']} fenetres OK, " + f"{statut['fenetres_vides_run']} vides, {statut['octets_run'] / 1e6:.1f} Mo gz") + return 0 + + +if __name__ == "__main__": + sys.exit(main() or 0)