Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
154 lignes
6.7 KiB
Python
154 lignes
6.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Archive les relevés des stations de vercors.me (SQLite, un point toutes les 10 minutes, jamais purgé)
|
|
et écrit les courbes des 7 derniers jours pour la page (public/data/semaine/<id>.json + index.json).
|
|
|
|
Entrées : les JSON publics des collecteurs (même format que meteo.json), lus dans l'ordre : le dernier fichier
|
|
l'emporte quand deux sources donnent le même point. Python standard uniquement, aucun appel réseau.
|
|
|
|
Usage :
|
|
python archiver_historique.py --db archive/vercors.db --sortie public/data/semaine \
|
|
--entree infoclimat.json --entree meteofrance.json:mf --entree weatherlink.json
|
|
« :mf » = les identifiants mf-xxx deviennent ceux du site (voir CORRESPONDANCE_MF).
|
|
"""
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
from collecter_weatherlink import ecrire_atomique # noqa: E402
|
|
|
|
PAS_ARCHIVE = 600 # un point par tranche de 10 minutes
|
|
PAS_SORTIE = 1800 # courbes de la page : un point toutes les 30 minutes
|
|
JOURS = 7
|
|
CORRESPONDANCE_MF = {"mf-autrans": "autrans", "mf-villard": "villard", "mf-chapelle": "chapelle",
|
|
"mf-rousset": "rousset", "mf-gresse": "gresse", "mf-st-jean": "st-jean"}
|
|
# colonne de la base <- champ du JSON public
|
|
CHAMPS = (("t", "temperature"), ("h", "humidite"), ("td", "point_de_rosee"), ("p", "pression"),
|
|
("v", "vent_moyen"), ("raf", "vent_rafales"), ("dir", "vent_direction"), ("r1", "pluie_1h"))
|
|
|
|
SCHEMA = """CREATE TABLE IF NOT EXISTS obs (
|
|
id TEXT NOT NULL, ts INTEGER NOT NULL,
|
|
t REAL, h REAL, td REAL, p REAL, v REAL, raf REAL, dir REAL, r1 REAL,
|
|
PRIMARY KEY (id, ts)) WITHOUT ROWID"""
|
|
|
|
|
|
# Stations sans anémomètre ni pluviomètre (la VP2 de Lans village enregistre 0 par défaut)
|
|
SANS_CAPTEUR = {"lans": ("v", "r")}
|
|
|
|
def epoch(dh_utc):
|
|
d = dt.datetime.strptime(dh_utc[:19].replace("T", " "), "%Y-%m-%d %H:%M:%S").replace(tzinfo=dt.timezone.utc)
|
|
return int(d.timestamp())
|
|
|
|
|
|
def ouvrir(chemin):
|
|
os.makedirs(os.path.dirname(os.path.abspath(chemin)), exist_ok=True)
|
|
db = sqlite3.connect(chemin)
|
|
db.execute(SCHEMA)
|
|
return db
|
|
|
|
|
|
def ingerer(db, source, correspondance=None):
|
|
"""Ajoute les relevés d'un JSON public. Une valeur vide ne remplace jamais une valeur déjà connue."""
|
|
n = 0
|
|
cols = [c for c, _ in CHAMPS]
|
|
sql = ("INSERT INTO obs (id, ts, %s) VALUES (?, ?, %s) ON CONFLICT(id, ts) DO UPDATE SET %s" %
|
|
(", ".join(cols), ", ".join("?" * len(cols)), ", ".join("%s = COALESCE(excluded.%s, %s)" % (c, c, c) for c in cols)))
|
|
for sid, st in (source.get("stations") or {}).items():
|
|
site = (correspondance or {}).get(sid, sid)
|
|
for r in st.get("releves") or []:
|
|
if not isinstance(r, dict) or not isinstance(r.get("dh_utc"), str) or r.get("temperature") is None:
|
|
continue
|
|
try:
|
|
ts = epoch(r["dh_utc"]) // PAS_ARCHIVE * PAS_ARCHIVE
|
|
except ValueError:
|
|
continue
|
|
vals = [r.get(champ) if isinstance(r.get(champ), (int, float)) else None for _, champ in CHAMPS]
|
|
db.execute(sql, [site, ts] + vals)
|
|
n += 1
|
|
db.commit()
|
|
return n
|
|
|
|
|
|
def moyenne(valeurs):
|
|
v = [x for x in valeurs if x is not None]
|
|
return round(sum(v) / len(v), 1) if v else None
|
|
|
|
|
|
def courbes(db, maintenant, jours=JOURS):
|
|
"""{id: courbes alignées sur des tranches de 30 min} pour les `jours` derniers jours."""
|
|
fin = maintenant // PAS_SORTIE * PAS_SORTIE
|
|
debut = fin - jours * 86400
|
|
n = (fin - debut) // PAS_SORTIE + 1
|
|
out = {}
|
|
for (sid,) in db.execute("SELECT DISTINCT id FROM obs").fetchall():
|
|
lignes = db.execute("SELECT ts, t, h, v, r1 FROM obs WHERE id = ? AND ts >= ? ORDER BY ts", (sid, debut)).fetchall()
|
|
if not lignes:
|
|
continue
|
|
cases = [[] for _ in range(n)]
|
|
pluie = [None] * n
|
|
for ts, t, h, v, r1 in lignes:
|
|
i = (ts - debut) // PAS_SORTIE
|
|
if 0 <= i < n:
|
|
cases[i].append((t, h, v))
|
|
if r1 is not None and ts % 3600 < PAS_ARCHIVE: # pluie des 60 dernières minutes, relevée à l'heure pile
|
|
pluie[i] = r1
|
|
courbe = {
|
|
"id": sid, "debut": debut, "pas": PAS_SORTIE,
|
|
"t": [moyenne(c[0] for c in cs) for cs in cases],
|
|
"h": [moyenne(c[1] for c in cs) for cs in cases],
|
|
"v": [moyenne(c[2] for c in cs) for cs in cases],
|
|
"r": pluie,
|
|
}
|
|
for grandeur in SANS_CAPTEUR.get(sid, ()): # station sans ce capteur : les zéros du logger ne sont pas des mesures
|
|
courbe[grandeur] = [None] * len(courbe[grandeur])
|
|
courbe["capteurs"] = {"vent": any(x is not None for x in courbe["v"]),
|
|
"pluie": any(x is not None for x in courbe["r"]),
|
|
"humidite": any(x is not None for x in courbe["h"])}
|
|
premier = db.execute("SELECT MIN(ts) FROM obs WHERE id = ?", (sid,)).fetchone()[0]
|
|
courbe["archive_depuis"] = premier
|
|
out[sid] = courbe
|
|
return out
|
|
|
|
|
|
def ecrire(dossier, maintenant, db):
|
|
cb = courbes(db, maintenant)
|
|
index = {}
|
|
for sid, c in cb.items():
|
|
ecrire_atomique(os.path.join(dossier, sid + ".json"), c)
|
|
index[sid] = {"vent": c["capteurs"]["vent"], "pluie": c["capteurs"]["pluie"],
|
|
"depuis": c["archive_depuis"], "points": sum(x is not None for x in c["t"])}
|
|
ecrire_atomique(os.path.join(dossier, "index.json"), index)
|
|
return index
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
|
ap.add_argument("--db", required=True)
|
|
ap.add_argument("--sortie", required=True)
|
|
ap.add_argument("--entree", action="append", default=[], help="FICHIER[:mf] ; plusieurs fois, le dernier l'emporte")
|
|
ap.add_argument("--dossier-entrees", default=".")
|
|
a = ap.parse_args(argv)
|
|
db = ouvrir(os.path.expanduser(a.db))
|
|
total = 0
|
|
for spec in a.entree:
|
|
chemin, _, option = spec.partition(":")
|
|
chemin = os.path.join(os.path.expanduser(a.dossier_entrees), chemin)
|
|
try:
|
|
with open(chemin, encoding="utf-8") as f:
|
|
source = json.load(f)
|
|
except (OSError, ValueError):
|
|
print("illisible : %s" % os.path.basename(chemin))
|
|
continue
|
|
total += ingerer(db, source, CORRESPONDANCE_MF if option == "mf" else None)
|
|
index = ecrire(os.path.expanduser(a.sortie), int(dt.datetime.now(dt.timezone.utc).timestamp()), db)
|
|
lignes = db.execute("SELECT COUNT(*) FROM obs").fetchone()[0]
|
|
print("archive : %d relevés lus, %d points en base, %d stations écrites" % (total, lignes, len(index)))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|