Fichiers
2026-10-11 01:54:18 +02:00

335 lignes
13 KiB
Python

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
fetch_dalles.py — telecharge un lot de dalles LiDAR HD IGN decrites par un JSON.
PHASE 9. La Phase 7 avait telecharge UNE dalle a la main ; ici il y en a 25, et
un telechargement de plusieurs gigaoctets qui echoue a la 23e sans rien dire
serait le pire resultat possible. D'ou : taille totale connue AVANT de commencer,
reprise sur octets deja recus, reessais avec attente croissante, et un bilan
final qui NOMME ce qui a echoue au lieu de s'arreter en silence.
python tools/fetch_dalles.py vercora_dalles.json -o data/lidar
python tools/fetch_dalles.py vercora_dalles.json -o data/lidar --only 0899-6440 0900-6440
python tools/fetch_dalles.py vercora_dalles.json -o data/lidar --dry-run
Options :
--only C [C ...] ne traiter que ces dalles (coordonnees_NW, ex. 0899-6440)
--dry-run interroge les tailles et affiche le bilan, sans rien ecrire
--retries N tentatives par dalle (defaut 4)
--margin-go N marge de disque libre exigee en plus du lot (defaut 5)
FORMAT D'ENTREE attendu (celui exporte depuis data.geopf.fr) : un objet dont
chaque valeur porte "url" et "metadata", cette derniere etant une CHAINE de JSON
a re-parser (l'IGN imbrique du JSON dans du JSON) contenant au moins
"coordonnees_NW" ("0899-6440" = coin nord-ouest en km Lambert 93) et
"nombre_points".
REPRISE : un fichier deja present et complet (taille == Content-Length) est
saute, pas retelecharge. Un fichier partiel est repris par requete Range si le
serveur l'accepte, sinon reparti de zero. C'est ce qui rend le script rejouable
apres une coupure sans repayer les gigaoctets deja recus.
"""
from __future__ import annotations
import argparse
import json
import os
import shutil
import sys
import time
import urllib.error
import urllib.request
USER_AGENT = "Vercora/0.9 (+projet de jeu personnel; laspy pipeline)"
TIMEOUT = 60
CHUNK = 1 << 20 # 1 Mo
def human(n: float) -> str:
for unit in ("o", "Ko", "Mo", "Go"):
if abs(n) < 1024 or unit == "Go":
return f"{n:.1f} {unit}" if unit != "o" else f"{int(n)} o"
n /= 1024.0
return f"{n:.1f} Go"
def human_time(seconds: float) -> str:
seconds = int(seconds)
if seconds < 60:
return f"{seconds} s"
if seconds < 3600:
return f"{seconds // 60} min {seconds % 60:02d} s"
return f"{seconds // 3600} h {(seconds % 3600) // 60:02d} min"
def load_manifest(path: str, only: list[str] | None):
"""Retourne [(coord, nom, url, nombre_points)] trie du nord-ouest au sud-est."""
with open(path, "r", encoding="utf-8") as f:
raw = json.load(f)
entries = []
for name, item in raw.items():
url = item["url"]
# L'IGN imbrique le bloc de metadonnees comme une CHAINE de JSON.
meta = item["metadata"]
if isinstance(meta, str):
meta = json.loads(meta)
coord = meta["coordonnees_NW"]
entries.append((coord, name, url, int(meta.get("nombre_points", 0))))
if only:
wanted = set(only)
kept = [e for e in entries if e[0] in wanted]
missing = wanted - {e[0] for e in kept}
if missing:
sys.exit(f"--only : dalle(s) absente(s) du manifeste : "
f"{', '.join(sorted(missing))}")
entries = kept
# Nord-ouest -> sud-est : X croissant, Y DECROISSANT (le nord est en haut).
def key(e):
x, y = e[0].split("-")
return (-int(y), int(x))
entries.sort(key=key)
return entries
def remote_size(url: str) -> int | None:
"""Content-Length sans telecharger le corps. None si le serveur se tait."""
request = urllib.request.Request(url, method="HEAD",
headers={"User-Agent": USER_AGENT})
try:
with urllib.request.urlopen(request, timeout=TIMEOUT) as response:
length = response.headers.get("Content-Length")
return int(length) if length is not None else None
except (urllib.error.URLError, ValueError, TimeoutError):
return None
def download_one(url: str, destination: str, expected: int | None,
retries: int, index: int, total: int, label: str) -> tuple[bool, str]:
"""Telecharge avec reprise et reessais. Retourne (succes, message)."""
temporary = destination + ".part"
for attempt in range(1, retries + 1):
have = os.path.getsize(temporary) if os.path.exists(temporary) else 0
if expected is not None and have == expected:
os.replace(temporary, destination)
return True, "deja complet (.part)"
if expected is not None and have > expected:
# Fichier partiel incoherent (manifeste change, disque corrompu) :
# mieux vaut repartir que concatener n'importe quoi.
os.remove(temporary)
have = 0
headers = {"User-Agent": USER_AGENT}
if have > 0:
headers["Range"] = f"bytes={have}-"
try:
request = urllib.request.Request(url, headers=headers)
with urllib.request.urlopen(request, timeout=TIMEOUT) as response:
# 200 sur une requete Range = le serveur ignore la reprise et
# renvoie tout : il faut alors reecrire depuis le debut, sinon
# on colle le fichier entier a la suite du morceau deja recu.
resuming = (response.status == 206)
if have > 0 and not resuming:
have = 0
mode = "ab" if resuming and have > 0 else "wb"
started = time.time()
last_print = 0.0
received = have
with open(temporary, mode) as f:
while True:
chunk = response.read(CHUNK)
if not chunk:
break
f.write(chunk)
received += len(chunk)
now = time.time()
if now - last_print >= 0.5:
last_print = now
elapsed = max(now - started, 1e-6)
speed = (received - have) / elapsed
if expected:
pct = 100.0 * received / expected
eta = ((expected - received) / speed
if speed > 0 else 0)
bar_width = 24
filled = int(bar_width * received / expected)
bar = "#" * filled + "." * (bar_width - filled)
sys.stdout.write(
f"\r [{index}/{total}] {label} "
f"[{bar}] {pct:5.1f}% "
f"{human(received)}/{human(expected)} "
f"{human(speed)}/s reste {human_time(eta)} ")
else:
sys.stdout.write(
f"\r [{index}/{total}] {label} "
f"{human(received)} {human(speed)}/s ")
sys.stdout.flush()
sys.stdout.write("\r" + " " * 100 + "\r")
sys.stdout.flush()
size = os.path.getsize(temporary)
if expected is not None and size != expected:
raise IOError(f"taille finale {size} != attendu {expected}")
# Un .laz commence par "LASF" : un HTML d'erreur enregistre sous le
# bon nom passerait sinon inapercu jusqu'a la conversion.
with open(temporary, "rb") as f:
magic = f.read(4)
if magic != b"LASF":
raise IOError(f"en-tete inattendu {magic!r} (pas un LAS/LAZ)")
os.replace(temporary, destination)
elapsed = time.time() - started
return True, f"{human(size)} en {human_time(elapsed)}"
except (urllib.error.URLError, IOError, TimeoutError, OSError) as error:
sys.stdout.write("\r" + " " * 100 + "\r")
sys.stdout.flush()
if attempt >= retries:
return False, f"{type(error).__name__}: {error}"
wait = min(2 ** attempt, 30)
print(f" [{index}/{total}] {label} : echec ({error}), "
f"nouvelle tentative {attempt + 1}/{retries} dans {wait} s")
time.sleep(wait)
return False, "tentatives epuisees"
def main() -> int:
parser = argparse.ArgumentParser(
description="Telecharge les dalles LiDAR HD decrites par un manifeste JSON.")
parser.add_argument("manifest")
parser.add_argument("-o", "--output", default="data/lidar")
parser.add_argument("--only", nargs="+", default=None)
parser.add_argument("--dry-run", action="store_true")
parser.add_argument("--retries", type=int, default=4)
parser.add_argument("--margin-go", type=float, default=5.0)
args = parser.parse_args()
entries = load_manifest(args.manifest, args.only)
print(f"Manifeste : {args.manifest} -> {len(entries)} dalle(s)")
coords = [e[0] for e in entries]
xs = sorted({int(c.split("-")[0]) for c in coords})
ys = sorted({int(c.split("-")[1]) for c in coords})
print(f" emprise (coins NO) : X {xs[0]}..{xs[-1]} km, Y {ys[0]}..{ys[-1]} km "
f"({len(xs)} x {len(ys)} dalles)")
print(f" points annonces : {sum(e[3] for e in entries):,}")
os.makedirs(args.output, exist_ok=True)
# --- Tailles reelles AVANT de telecharger quoi que ce soit ---------------
print(f"\nInterrogation des tailles ({len(entries)} requetes HEAD)...")
sizes: dict[str, int | None] = {}
unknown = 0
for index, (coord, name, url, points) in enumerate(entries, 1):
size = remote_size(url)
sizes[coord] = size
if size is None:
unknown += 1
sys.stdout.write(f"\r {index}/{len(entries)} ")
sys.stdout.flush()
print("\r" + " " * 40 + "\r", end="")
known = [s for s in sizes.values() if s is not None]
total_known = sum(known)
if unknown and known:
# Estimation pour les manquantes, a partir de la moyenne des connues.
average = total_known / len(known)
estimated_total = total_known + unknown * average
else:
estimated_total = total_known
print(f" taille totale : {human(total_known)}"
+ (f" (+{unknown} dalle(s) de taille inconnue, "
f"estimation totale {human(estimated_total)})" if unknown else ""))
already = 0
todo = []
for coord, name, url, points in entries:
destination = os.path.join(args.output, os.path.basename(url))
size = sizes[coord]
if os.path.exists(destination) and (size is None
or os.path.getsize(destination) == size):
already += os.path.getsize(destination)
continue
todo.append((coord, name, url, destination, size))
partial = sum(os.path.getsize(t[3] + ".part")
for t in todo if os.path.exists(t[3] + ".part"))
remaining = max(estimated_total - already - partial, 0)
print(f" deja sur disque : {human(already)}"
+ (f" (+{human(partial)} de fragments repris)" if partial else ""))
print(f" a telecharger : {human(remaining)} ({len(todo)} dalle(s))")
# --- Verification du disque ---------------------------------------------
free = shutil.disk_usage(os.path.abspath(args.output)).free
needed = remaining + args.margin_go * (1 << 30)
print(f"\nDisque : {human(free)} libres, besoin {human(remaining)} "
f"+ {args.margin_go:g} Go de marge = {human(needed)}")
if free < needed:
print(" INSUFFISANT — rien n'a ete telecharge.", file=sys.stderr)
return 2
print(" OK")
if args.dry_run:
print("\n--dry-run : rien n'a ete telecharge.")
return 0
if not todo:
print("\nToutes les dalles sont deja presentes. Rien a faire.")
return 0
# --- Telechargement ------------------------------------------------------
print(f"\nTelechargement de {len(todo)} dalle(s) vers {args.output}/\n")
started = time.time()
failures = []
done_bytes = 0
for index, (coord, name, url, destination, size) in enumerate(todo, 1):
ok, message = download_one(url, destination, size, args.retries,
index, len(todo), coord)
if ok:
done_bytes += os.path.getsize(destination)
elapsed = time.time() - started
speed = done_bytes / elapsed if elapsed > 0 else 0
left = max(remaining - done_bytes, 0)
eta = left / speed if speed > 0 else 0
print(f" [{index}/{len(todo)}] {coord} OK {message}"
f" | lot : {100.0 * index / len(todo):.0f}%, "
f"reste ~{human_time(eta)}")
else:
failures.append((coord, message))
print(f" [{index}/{len(todo)}] {coord} ECHEC {message}")
elapsed = time.time() - started
print(f"\nTermine en {human_time(elapsed)} — "
f"{len(todo) - len(failures)}/{len(todo)} reussie(s), "
f"{human(done_bytes)} recus"
+ (f", {human(done_bytes / elapsed)}/s de moyenne" if elapsed > 0 else ""))
if failures:
print(f"\n{len(failures)} ECHEC(S) — relancer la meme commande reprendra "
f"ces dalles la ou elles en sont :")
for coord, message in failures:
print(f" {coord} : {message}")
return 1
return 0
if __name__ == "__main__":
sys.exit(main())