609 lignes
24 KiB
Python
609 lignes
24 KiB
Python
#!/usr/bin/env python3
|
|
"""Collecte automatisee de l'historique Infoclimat (stdlib uniquement).
|
|
|
|
Usage :
|
|
python3 collecte.py # tourne jusqu'au plafond du jour ou la fin de la file
|
|
python3 collecte.py --dry-run # construit la file, affiche les 5 premieres fenetres, aucun appel API
|
|
python3 collecte.py --max-requetes N # s'arrete apres N appels HTTP (test)
|
|
python3 collecte.py --maj # mise a jour quotidienne des stations finies (derniere date + 1 -> hier)
|
|
python3 collecte.py --maj --dry-run # fenetres de mise a jour prevues, aucun appel API
|
|
Voir README.md.
|
|
"""
|
|
import argparse, datetime as dt, fcntl, gzip, json, os, re, shutil, socket, sys, time, zlib
|
|
import urllib.error, urllib.parse, urllib.request
|
|
from pathlib import Path
|
|
from zoneinfo import ZoneInfo
|
|
|
|
BASE = Path(__file__).resolve().parent
|
|
DONNEES = BASE / "donnees"
|
|
TOKEN_F = BASE / "infoclimat.token"
|
|
STATIONS_F = BASE / "stations.json"
|
|
ETAT_F = BASE / "etat.json"
|
|
FENETRES_F = BASE / "fenetres.jsonl"
|
|
QUOTA_F = BASE / "quota.json"
|
|
STATUT_F = BASE / "statut.json"
|
|
LOG_F = BASE / "collecte.log"
|
|
STOP_F = BASE / "STOP_RAISON.txt"
|
|
LOCK_F = BASE / ".collecte.lock"
|
|
|
|
PARIS = ZoneInfo("Europe/Paris")
|
|
# PROPOSITION 2026-09-30 : 4500 -> 4999. Marge de securite volontaire (1 requete
|
|
# sous une limite reelle supposee de 5000/jour) car le comportement exact du
|
|
# quota Infoclimat (valeur exacte, heure de reinitialisation, fenetre glissante
|
|
# ou compteur calendaire) n'est PAS confirme officiellement. A surveiller apres
|
|
# coup via quota.json et les entrees "limite"/HTTP 429/503 de collecte.log ;
|
|
# revenir a une valeur plus basse si des 429/503 apparaissent avant la fin de journee.
|
|
PLAFOND = 4999
|
|
PAUSE = 1.2 # s entre deux debuts de requete (>= 1,1 s)
|
|
TIMEOUT = 300
|
|
UA = "GastinelWeatherStudio/0.4 (collecte historique)"
|
|
PLANCHER = dt.date(2004, 1, 1)
|
|
MAX_DONNEES = 8 * 1024**3
|
|
MAX_DISQUE = 0.85
|
|
LOG_MAX = 5 * 1024**2
|
|
MAJ_MAX = 1200 # requetes max par jour pour --maj ; aussi reservees par le run de nuit
|
|
|
|
TOKEN = ""
|
|
_last = [0.0]
|
|
statut = {"debut_run": None, "derniere_erreur": None, "octets_run": 0, "fenetres_ok_run": 0,
|
|
"fenetres_vides_run": 0, "requetes_run": 0, "etat": "demarrage"}
|
|
|
|
|
|
# ---------------------------------------------------------------- utilitaires
|
|
def masquer(s):
|
|
s = str(s)
|
|
if TOKEN:
|
|
s = s.replace(TOKEN, "***").replace(urllib.parse.quote(TOKEN), "***")
|
|
return re.sub(r"token=[^&\s\"']+", "token=***", s)
|
|
|
|
|
|
def log(msg):
|
|
ligne = f"{dt.datetime.now(PARIS):%Y-%m-%d %H:%M:%S} {masquer(msg)}"
|
|
print(ligne, flush=True)
|
|
try:
|
|
if LOG_F.exists() and LOG_F.stat().st_size > LOG_MAX:
|
|
LOG_F.replace(LOG_F.with_suffix(".log.1"))
|
|
with open(LOG_F, "a", encoding="utf-8") as f:
|
|
f.write(ligne + "\n")
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def ecrire_json(path, obj):
|
|
tmp = path.with_suffix(path.suffix + ".tmp")
|
|
tmp.write_text(json.dumps(obj, ensure_ascii=False, indent=1), encoding="utf-8")
|
|
os.replace(tmp, path)
|
|
|
|
|
|
def lire_json(path, defaut):
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, ValueError):
|
|
return defaut
|
|
|
|
|
|
def aujourdhui():
|
|
return dt.datetime.now(PARIS).date()
|
|
|
|
|
|
def d2s(d):
|
|
return d.isoformat()
|
|
|
|
|
|
def s2d(s):
|
|
return dt.date.fromisoformat(s)
|
|
|
|
|
|
# ---------------------------------------------------------------- quota
|
|
def quota_lire():
|
|
q = lire_json(QUOTA_F, {})
|
|
j = d2s(aujourdhui())
|
|
if q.get("date") != j:
|
|
q = {"date": j, "requetes": 0}
|
|
return q
|
|
|
|
|
|
def quota_incr():
|
|
q = quota_lire()
|
|
q["requetes"] += 1
|
|
ecrire_json(QUOTA_F, q)
|
|
return q["requetes"]
|
|
|
|
|
|
def quota_restant():
|
|
return PLAFOND - quota_lire()["requetes"]
|
|
|
|
|
|
# ---------------------------------------------------------------- disque
|
|
def taille_donnees():
|
|
t = 0
|
|
for racine, _, fichiers in os.walk(DONNEES):
|
|
for f in fichiers:
|
|
try:
|
|
t += os.path.getsize(os.path.join(racine, f))
|
|
except OSError:
|
|
pass
|
|
return t
|
|
|
|
|
|
def disque_ok():
|
|
u = shutil.disk_usage("/")
|
|
frac = u.used / u.total
|
|
td = taille_donnees()
|
|
if td > MAX_DONNEES or frac > MAX_DISQUE:
|
|
return False, f"donnees={td / 1e9:.2f} Go, disque /={frac:.0%}"
|
|
return True, ""
|
|
|
|
|
|
# ---------------------------------------------------------------- file de travail
|
|
def init_etat():
|
|
"""Etat par station ; construit une fois, conserve ensuite (reprise)."""
|
|
etat = lire_json(ETAT_F, None)
|
|
if etat:
|
|
return etat
|
|
st = lire_json(STATIONS_F, [])
|
|
fin = d2s(aujourdhui() - dt.timedelta(days=1))
|
|
ordre = sorted(st, key=lambda s: (not s.get("donnees_recentes"), not s.get("date_debut"), s["id"]))
|
|
stations = {}
|
|
for s in ordre:
|
|
pas = s.get("pas_minutes")
|
|
active = bool(s.get("donnees_recentes"))
|
|
if s.get("date_debut"):
|
|
mode, cur = "avant", s["date_debut"] + "-01"
|
|
taille = 365 if (pas and pas >= 30) else 180
|
|
elif active:
|
|
mode, cur, taille = "arriere", fin, 365 if (pas and pas >= 30) else 180
|
|
else:
|
|
mode, cur, taille = "avant", d2s(PLANCHER), 365
|
|
stations[s["id"]] = {"mode": mode, "curseur": cur, "fin": fin, "taille": taille,
|
|
"active": active, "vides_consec": 0, "fini": False, "erreurs": 0,
|
|
"fenetres_ok": 0, "fenetres_vides": 0, "octets": 0}
|
|
etat = {"cree": dt.datetime.now(PARIS).isoformat(timespec="seconds"),
|
|
"ordre": [s["id"] for s in ordre], "stations": stations}
|
|
ecrire_json(ETAT_F, etat)
|
|
return etat
|
|
|
|
|
|
def prochaine_fenetre(e):
|
|
cur, fin, t = s2d(e["curseur"]), s2d(e["fin"]), e["taille"]
|
|
if e["mode"] == "avant":
|
|
if cur > fin:
|
|
return None
|
|
return cur, min(cur + dt.timedelta(days=t - 1), fin)
|
|
if cur < PLANCHER:
|
|
return None
|
|
return max(cur - dt.timedelta(days=t - 1), PLANCHER), cur
|
|
|
|
|
|
def avancer(e, a, b, vide):
|
|
if e["mode"] == "avant":
|
|
e["curseur"] = d2s(b + dt.timedelta(days=1))
|
|
if s2d(e["curseur"]) > s2d(e["fin"]):
|
|
e["fini"] = True
|
|
else:
|
|
e["vides_consec"] = e["vides_consec"] + 1 if vide else 0
|
|
e["curseur"] = d2s(a - dt.timedelta(days=1))
|
|
if e["vides_consec"] >= 2 or a <= PLANCHER:
|
|
e["fini"] = True
|
|
e["erreurs"] = 0
|
|
|
|
|
|
def noter(sid, a, b, resultat, **kw):
|
|
with open(FENETRES_F, "a", encoding="utf-8") as f:
|
|
f.write(json.dumps({"id": sid, "start": d2s(a), "end": d2s(b), "res": resultat,
|
|
"t": dt.datetime.now(PARIS).isoformat(timespec="seconds"), **kw}) + "\n")
|
|
|
|
|
|
# ---------------------------------------------------------------- appel API
|
|
class Arret(Exception):
|
|
pass
|
|
|
|
|
|
def appeler(sid, a, b):
|
|
"""Retourne (code, info). code : ok | vide | trop | timeout | limite | erreur."""
|
|
url = ("https://www.infoclimat.fr/opendata/?version=2&method=get&format=json&stations[]="
|
|
+ urllib.parse.quote(sid) + f"&start={a}&end={b}&token=" + urllib.parse.quote(TOKEN))
|
|
attente = PAUSE - (time.time() - _last[0])
|
|
if attente > 0:
|
|
time.sleep(attente)
|
|
_last[0] = time.time()
|
|
n = quota_incr()
|
|
statut["requetes_run"] += 1
|
|
rep = DONNEES / sid
|
|
rep.mkdir(parents=True, exist_ok=True)
|
|
final = rep / f"{a}_{b}.json.gz"
|
|
part = rep / f"{a}_{b}.json.gz.part"
|
|
req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Encoding": "gzip"})
|
|
t0 = time.time()
|
|
brut = 0
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
|
|
deja_gz = (r.headers.get("Content-Encoding") or "").lower() == "gzip"
|
|
with open(part, "wb") as fo:
|
|
sortie = fo if deja_gz else gzip.GzipFile(fileobj=fo, mode="wb", compresslevel=6)
|
|
while True:
|
|
if time.time() - t0 > TIMEOUT:
|
|
raise socket.timeout("duree totale > timeout")
|
|
x = r.read(1 << 16)
|
|
if not x:
|
|
break
|
|
brut += len(x)
|
|
sortie.write(x)
|
|
if not deja_gz:
|
|
sortie.close()
|
|
except urllib.error.HTTPError as e:
|
|
part.unlink(missing_ok=True)
|
|
corps = masquer(e.read(2000).decode("utf-8", "replace"))
|
|
if e.code in (429, 503):
|
|
return "limite", f"HTTP {e.code}"
|
|
return analyser_texte(corps) or ("erreur", f"HTTP {e.code}: {corps[:150]}")
|
|
except (socket.timeout, TimeoutError) as e:
|
|
part.unlink(missing_ok=True)
|
|
return "timeout", masquer(e)
|
|
except (urllib.error.URLError, OSError, zlib.error) as e:
|
|
part.unlink(missing_ok=True)
|
|
return "erreur", f"{type(e).__name__}: {masquer(e)[:150]}"
|
|
duree = time.time() - t0
|
|
code, info = valider(part)
|
|
if code == "ok":
|
|
os.replace(part, final)
|
|
taille = final.stat().st_size
|
|
statut["octets_run"] += taille
|
|
return "ok", {"lignes": info, "gz": taille, "duree": round(duree, 1), "quota": n}
|
|
part.unlink(missing_ok=True)
|
|
return code, info
|
|
|
|
|
|
def analyser_texte(txt):
|
|
bas = txt.lower()
|
|
if "wrong ip" in bas:
|
|
return "fatal", "Wrong ip address"
|
|
if "no account found" in bas:
|
|
return "fatal", "No account found"
|
|
if "too many lines" in bas:
|
|
return "trop", "Too many lines"
|
|
if "too many requests" in bas:
|
|
return "limite", "Too many requests"
|
|
return None
|
|
|
|
|
|
def valider(part):
|
|
"""Relit le gz en flux : JSON objet, status OK, pas d'erreur, compte les dh_utc."""
|
|
try:
|
|
with gzip.open(part, "rb") as f:
|
|
tete = f.read(4096)
|
|
txt = tete.decode("utf-8", "replace")
|
|
a = analyser_texte(txt[:2000])
|
|
if a:
|
|
return a
|
|
if not txt.lstrip().startswith("{"):
|
|
return "erreur", "reponse non JSON: " + masquer(txt[:150])
|
|
m = re.search(r'"status"\s*:\s*"([^"]*)"', txt)
|
|
if not m or m.group(1).upper() != "OK":
|
|
err = re.search(r'"errors"\s*:\s*\[\s*"([^"]+)"', txt)
|
|
message = err.group(1) if err else txt[:150]
|
|
# PROPOSITION 2026-09-30 : le forum Infoclimat signale des reponses HTTP 200
|
|
# avec un corps JSON status != "OK" pour indiquer un quota depasse (au lieu
|
|
# d'un 429/503). On reutilise le champ d'erreur deja lu ci-dessus (structure
|
|
# {"status":..., "errors":[...]} deja geree par ce code) et on route vers
|
|
# "limite" (meme pause de 15 min que 429/503, cf. appeler()/traiter()) des
|
|
# que le message evoque une limitation, plutot que "erreur" (qui ne fait que
|
|
# quelques essais rapides avant abandon et pourrait aggraver le depassement).
|
|
# HYPOTHESE NON CONFIRMEE : le libelle exact utilise par Infoclimat en cas de
|
|
# quota depasse avec HTTP 200 n'a pas ete observe dans collecte.log a ce jour
|
|
# (grep -i "erreur" collecte.log ne montre que des HTTP 502 et "JSON tronque").
|
|
# A VERIFIER : des qu'un cas reel apparait, comparer son libelle avec la liste
|
|
# de mots-cles ci-dessous et l'completer/corriger si besoin.
|
|
bas = message.lower()
|
|
if any(mot in bas for mot in ("quota", "limit", "trop de requ", "too many", "rate")):
|
|
return "limite", f"status non OK (HTTP 200) : {masquer(message)}"
|
|
return "erreur", "status non OK: " + masquer(message)
|
|
n = tete.count(b'"dh_utc"')
|
|
reste = tete[-16:]
|
|
dernier = tete
|
|
while True:
|
|
x = f.read(1 << 20)
|
|
if not x:
|
|
break
|
|
n += (reste + x).count(b'"dh_utc"')
|
|
reste = x[-16:]
|
|
dernier = x
|
|
if not dernier.rstrip().endswith(b"}"):
|
|
return "erreur", "JSON tronque"
|
|
return ("ok", n) if n else ("vide", 0)
|
|
except (OSError, EOFError, zlib.error) as e:
|
|
return "erreur", f"gz invalide: {e}"
|
|
|
|
|
|
# ---------------------------------------------------------------- statut
|
|
def maj_statut(etat, courant=None):
|
|
st = etat["stations"]
|
|
finies = [s for s in etat["ordre"] if st[s]["fini"]]
|
|
q = quota_lire()
|
|
if "derniere_maj" not in statut:
|
|
statut["derniere_maj"] = lire_json(STATUT_F, {}).get("derniere_maj")
|
|
statut.update({
|
|
"maj": dt.datetime.now(PARIS).isoformat(timespec="seconds"),
|
|
"stations_total": len(st), "stations_finies": len(finies),
|
|
"stations_restantes": len(st) - len(finies), "station_en_cours": courant,
|
|
"curseur_en_cours": st[courant]["curseur"] if courant else None,
|
|
"requetes_du_jour": q["requetes"], "plafond": PLAFOND, "jour_quota": q["date"],
|
|
"octets_total_collectes": sum(s["octets"] for s in st.values()),
|
|
"fenetres_ok_total": sum(s["fenetres_ok"] for s in st.values()),
|
|
"fenetres_vides_total": sum(s["fenetres_vides"] for s in st.values()),
|
|
})
|
|
ecrire_json(STATUT_F, statut)
|
|
|
|
|
|
def stop(raison):
|
|
STOP_F.write_text(f"{dt.datetime.now(PARIS).isoformat(timespec='seconds')} {raison}\n"
|
|
"Collecte arretee. Verifier la cle / l'IP puis supprimer ce fichier pour relancer.\n",
|
|
encoding="utf-8")
|
|
log(f"ARRET D'URGENCE : {raison}")
|
|
statut["derniere_erreur"] = raison
|
|
statut["etat"] = "arret_urgence"
|
|
|
|
|
|
# ---------------------------------------------------------------- boucle
|
|
def a_mettre_a_jour(etat):
|
|
"""Stations actives finies dont la derniere date collectee est < hier : (sid, debut, hier)."""
|
|
hier = aujourdhui() - dt.timedelta(days=1)
|
|
res = []
|
|
for sid in etat["ordre"]:
|
|
e = etat["stations"][sid]
|
|
if not (e["fini"] and e["active"]):
|
|
continue
|
|
debut = s2d(e.get("derniere", e["fin"])) + dt.timedelta(days=1)
|
|
if debut <= hier:
|
|
res.append((sid, debut, hier))
|
|
return res
|
|
|
|
|
|
def reserve_maj(etat):
|
|
"""Requetes laissees par le run de nuit pour la maj de 08:00 du meme jour Paris."""
|
|
return min(MAJ_MAX, len(a_mettre_a_jour(etat)) + 20)
|
|
|
|
|
|
def traiter(etat, max_req):
|
|
st = etat["stations"]
|
|
reserve = reserve_maj(etat)
|
|
log(f"Reserve pour la maj de 08:00 : {reserve} requetes (run de nuit plafonne a {PLAFOND - reserve})")
|
|
limites = 0
|
|
nreq = 0
|
|
for sid in etat["ordre"]:
|
|
e = st[sid]
|
|
while not e["fini"]:
|
|
if STOP_F.exists():
|
|
raise Arret("STOP_RAISON.txt present")
|
|
if quota_restant() <= reserve:
|
|
log(f"Plafond {PLAFOND} (- reserve maj {reserve}) requetes atteint pour le {quota_lire()['date']} : arret propre.")
|
|
statut["etat"] = "plafond_atteint"
|
|
return
|
|
if max_req and nreq >= max_req:
|
|
log(f"--max-requetes {max_req} atteint : arret.")
|
|
statut["etat"] = "arret_max_requetes"
|
|
return
|
|
if nreq % 25 == 0:
|
|
ok, pourquoi = disque_ok()
|
|
while not ok:
|
|
statut["etat"] = "pause_disque"
|
|
maj_statut(etat, sid)
|
|
log(f"Pause disque ({pourquoi}) : attente 30 min (rapatriement).")
|
|
time.sleep(1800)
|
|
ok, pourquoi = disque_ok()
|
|
statut["etat"] = "en_cours"
|
|
fen = prochaine_fenetre(e)
|
|
if not fen:
|
|
e["fini"] = True
|
|
break
|
|
a, b = fen
|
|
code, info = appeler(sid, a, b)
|
|
nreq += 1
|
|
if code == "ok":
|
|
limites = 0
|
|
e["fenetres_ok"] += 1
|
|
e["octets"] += info["gz"]
|
|
statut["fenetres_ok_run"] += 1
|
|
noter(sid, a, b, "ok", **info)
|
|
log(f"{sid} {a}..{b} OK {info['lignes']} lignes {info['gz'] / 1e6:.2f} Mo gz "
|
|
f"{info['duree']} s (quota {info['quota']})")
|
|
avancer(e, a, b, False)
|
|
elif code == "vide":
|
|
limites = 0
|
|
e["fenetres_vides"] += 1
|
|
statut["fenetres_vides_run"] += 1
|
|
noter(sid, a, b, "vide")
|
|
log(f"{sid} {a}..{b} vide")
|
|
avancer(e, a, b, True)
|
|
elif code in ("trop", "timeout"):
|
|
if e["taille"] > 1:
|
|
e["taille"] = max(1, e["taille"] // 2)
|
|
log(f"{sid} {a}..{b} {code} -> fenetre reduite a {e['taille']} j")
|
|
else:
|
|
e["erreurs"] += 1
|
|
if e["erreurs"] >= 3:
|
|
noter(sid, a, b, "abandon", info=str(info))
|
|
log(f"{sid} {a}..{b} {code} sur 1 jour x3 : fenetre abandonnee")
|
|
avancer(e, a, b, False)
|
|
elif code == "limite":
|
|
limites += 1
|
|
statut["derniere_erreur"] = f"{sid} {a}..{b} {info}"
|
|
if limites >= 4:
|
|
log("Limitation serveur persistante : fin du run (reprise au prochain).")
|
|
statut["etat"] = "limitation_serveur"
|
|
return
|
|
log(f"{info} : pause 15 min")
|
|
maj_statut(etat, sid)
|
|
time.sleep(900)
|
|
elif code == "fatal":
|
|
stop(info)
|
|
raise Arret(info)
|
|
else:
|
|
e["erreurs"] += 1
|
|
statut["derniere_erreur"] = f"{sid} {a}..{b} {info}"
|
|
log(f"{sid} {a}..{b} ERREUR {info} (essai {e['erreurs']})")
|
|
if e["erreurs"] >= 3:
|
|
noter(sid, a, b, "erreur", info=str(info))
|
|
avancer(e, a, b, False)
|
|
if not e["fini"] and e["mode"] == "arriere":
|
|
pass
|
|
else:
|
|
time.sleep(10 * e["erreurs"])
|
|
ecrire_json(ETAT_F, etat)
|
|
maj_statut(etat, sid)
|
|
ecrire_json(ETAT_F, etat)
|
|
statut["etat"] = "termine"
|
|
log("File de travail terminee.")
|
|
|
|
|
|
def traiter_maj(etat, max_req):
|
|
"""Une requete par station finie : (derniere date + 1) .. hier. Quota partage, <= MAJ_MAX."""
|
|
st = etat["stations"]
|
|
liste = a_mettre_a_jour(etat)
|
|
log(f"Maj : {len(liste)} stations a mettre a jour")
|
|
nreq = limites = 0
|
|
for sid, a, b in liste:
|
|
e = st[sid]
|
|
while True:
|
|
if STOP_F.exists():
|
|
raise Arret("STOP_RAISON.txt present")
|
|
if quota_restant() <= 0 or nreq >= min(MAJ_MAX, max_req or MAJ_MAX):
|
|
log(f"Maj : limite atteinte ({nreq} requetes maj, quota du jour "
|
|
f"{quota_lire()['requetes']}/{PLAFOND}) : reprise demain.")
|
|
statut["etat"] = "maj_plafond"
|
|
return
|
|
if nreq % 25 == 0:
|
|
ok, pourquoi = disque_ok()
|
|
if not ok:
|
|
log(f"Maj : disque plein ({pourquoi}) : arret, reprise demain.")
|
|
statut["etat"] = "maj_pause_disque"
|
|
return
|
|
code, info = appeler(sid, a, b)
|
|
nreq += 1
|
|
if code == "limite":
|
|
limites += 1
|
|
statut["derniere_erreur"] = f"maj {sid} {a}..{b} {info}"
|
|
if limites >= 4:
|
|
log("Maj : limitation serveur persistante : fin.")
|
|
statut["etat"] = "limitation_serveur"
|
|
return
|
|
log(f"{info} : pause 15 min")
|
|
time.sleep(900)
|
|
continue
|
|
if code == "fatal":
|
|
stop(info)
|
|
raise Arret(info)
|
|
limites = 0
|
|
if code == "ok":
|
|
e["fenetres_ok"] += 1
|
|
e["octets"] += info["gz"]
|
|
statut["fenetres_ok_run"] += 1
|
|
noter(sid, a, b, "ok", maj=True, **info)
|
|
log(f"maj {sid} {a}..{b} OK {info['lignes']} lignes (quota {info['quota']})")
|
|
e["derniere"] = d2s(b)
|
|
elif code == "vide":
|
|
e["fenetres_vides"] += 1
|
|
statut["fenetres_vides_run"] += 1
|
|
noter(sid, a, b, "vide", maj=True)
|
|
log(f"maj {sid} {a}..{b} vide")
|
|
e["derniere"] = d2s(b)
|
|
else: # erreur / trop / timeout : on ne bouge pas, la fenetre sera retentee demain (elargie)
|
|
statut["derniere_erreur"] = f"maj {sid} {a}..{b} {info}"
|
|
noter(sid, a, b, "erreur", maj=True, info=str(info))
|
|
log(f"maj {sid} {a}..{b} {code} {info} : reessai demain")
|
|
ecrire_json(ETAT_F, etat)
|
|
break
|
|
statut["etat"] = "maj_terminee"
|
|
log("Maj terminee.")
|
|
|
|
|
|
def dry_run_maj(etat):
|
|
liste = a_mettre_a_jour(etat)
|
|
q = quota_lire()
|
|
print(f"Maj : {len(liste)} stations finies a mettre a jour ; quota du jour ({q['date']}) "
|
|
f"{q['requetes']}/{PLAFOND} ; max {MAJ_MAX} requetes maj/jour ; reserve de nuit {reserve_maj(etat)}")
|
|
for sid, a, b in liste[:10]:
|
|
print(f" {sid} {a}..{b} ({(b - a).days + 1} j) -> donnees/{sid}/{a}_{b}.json.gz")
|
|
if len(liste) > 10:
|
|
print(f" ... +{len(liste) - 10} autres")
|
|
|
|
|
|
def dry_run(etat):
|
|
st = etat["stations"]
|
|
modes = {}
|
|
for s in st.values():
|
|
k = f"{'active' if s['active'] else 'arretee'}/{s['mode']}/{s['taille']}j"
|
|
modes[k] = modes.get(k, 0) + 1
|
|
print(f"Stations : {len(st)}, finies : {sum(s['fini'] for s in st.values())}")
|
|
for k, v in sorted(modes.items()):
|
|
print(f" {k:28s} {v}")
|
|
print(f"Quota du jour ({quota_lire()['date']}) : {quota_lire()['requetes']}/{PLAFOND}")
|
|
print(f"Disque : {disque_ok()}")
|
|
print("5 premieres fenetres (sans appel API) :")
|
|
n = 0
|
|
for sid in etat["ordre"]:
|
|
e = dict(st[sid])
|
|
while n < 5 and not e["fini"]:
|
|
f = prochaine_fenetre(e)
|
|
if not f:
|
|
break
|
|
print(f" {sid} {f[0]}..{f[1]} (mode {e['mode']}, {e['taille']} j)")
|
|
avancer(e, f[0], f[1], False)
|
|
n += 1
|
|
if n >= 5:
|
|
break
|
|
|
|
|
|
def main():
|
|
global TOKEN
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--dry-run", action="store_true")
|
|
ap.add_argument("--max-requetes", type=int, default=0)
|
|
ap.add_argument("--maj", action="store_true")
|
|
args = ap.parse_args()
|
|
os.chdir(BASE)
|
|
DONNEES.mkdir(exist_ok=True)
|
|
etat = init_etat()
|
|
if args.dry_run:
|
|
return dry_run_maj(etat) if args.maj else dry_run(etat)
|
|
if STOP_F.exists():
|
|
print(f"STOP_RAISON.txt present : rien n'est lance.\n{STOP_F.read_text()}")
|
|
return 2
|
|
lock = open(LOCK_F, "w")
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except OSError:
|
|
print("Une collecte tourne deja.")
|
|
return 0
|
|
TOKEN = TOKEN_F.read_text(encoding="utf-8").strip()
|
|
for p in DONNEES.rglob("*.part"):
|
|
p.unlink(missing_ok=True)
|
|
statut["debut_run"] = dt.datetime.now(PARIS).isoformat(timespec="seconds")
|
|
statut["etat"] = "en_cours"
|
|
if args.maj:
|
|
statut["derniere_maj"] = statut["debut_run"]
|
|
log(f"Debut du {'run MAJ' if args.maj else 'run'} (quota du jour {quota_lire()['requetes']}/{PLAFOND})")
|
|
try:
|
|
(traiter_maj if args.maj else traiter)(etat, args.max_requetes)
|
|
except Arret as e:
|
|
log(f"Arret : {e}")
|
|
return 3
|
|
except KeyboardInterrupt:
|
|
statut["etat"] = "interrompu"
|
|
log("Interrompu.")
|
|
except Exception as e:
|
|
statut["etat"] = "plante"
|
|
statut["derniere_erreur"] = masquer(f"{type(e).__name__}: {e}")
|
|
log(f"PLANTAGE {statut['derniere_erreur']}")
|
|
raise
|
|
finally:
|
|
ecrire_json(ETAT_F, etat)
|
|
maj_statut(etat)
|
|
log(f"Fin du run : {statut['requetes_run']} requetes, {statut['fenetres_ok_run']} fenetres OK, "
|
|
f"{statut['fenetres_vides_run']} vides, {statut['octets_run'] / 1e6:.1f} Mo gz")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main() or 0)
|