Fichiers
2026-10-11 01:57:14 +02:00

66 lignes
3.9 KiB
Python

# Phase 4 : descriptions issues de Wikipedia FR (REST summary), validees par coordonnees. Ecrit vercors_poi_complements.json
import json, os, re, math, time, urllib.request, urllib.parse
H = os.path.dirname(os.path.abspath(__file__)); F = os.path.join(H, 'vercors_poi_complements.json')
P = json.load(open(os.path.join(H, 'vercors_poi.json'), encoding='utf8'))
cur = json.load(open(F, encoding='utf8')) if os.path.exists(F) else {}
UA = {'User-Agent': 'VercorsInventaire/1.0 (usage interne; contact local)'}
def get(u):
import subprocess
for i in range(4):
try:
r = subprocess.run(['curl', '-s', '--max-time', '15', '-w', '\n%{http_code}', '-A', UA['User-Agent'], u], capture_output=True, timeout=25)
body, _, code = r.stdout.decode('utf8', 'replace').rpartition('\n')
if code == '200': return json.loads(body)
if code == '404': return None
time.sleep(30 if code == '429' else 3)
except Exception: time.sleep(3)
def dist(a, b, c, d):
return 6371*2*math.asin(math.sqrt(math.sin(math.radians(c-a)/2)**2+math.cos(math.radians(a))*math.cos(math.radians(c))*math.sin(math.radians(d-b)/2)**2))
def summ(t):
time.sleep(1.2)
return get('https://fr.wikipedia.org/api/rest_v1/page/summary/' + urllib.parse.quote(t.replace(' ', '_'), safe=''))
def search(q):
time.sleep(1.2)
r = get('https://fr.wikipedia.org/w/rest.php/v1/search/title?limit=2&q=' + urllib.parse.quote(q))
return [x['title'] for x in (r or {}).get('pages', [])]
def short(t):
t = re.sub(r'\s*\([^)]*\)', '', t) if len(t) > 400 else t
s = re.split(r'(?<=[.!?])\s+(?=[A-ZÉÈÀ])', t)
out = ''
for x in s:
if len(out) + len(x) > 380 and out: break
out += (' ' if out else '') + x
if len(out) > 200: break
return out
targets = [p for p in P if p['id'] not in cur and (
(p['categorie'] == 'lieu_habite' and p['type'] in ('village', 'ville')) or (p['categorie'] == 'patrimoine' and not p.get('description')))]
targets.sort(key=lambda p: p['categorie'] != 'lieu_habite')
print(len(targets), 'cibles')
ok = 0
for i, p in enumerate(targets):
if p['categorie'] == 'lieu_habite' and p['type'] not in ('village','ville'):
if not any('wikipedia' in s for s in p['sources']) and len(p['nom']) < 6: continue
titles = [urllib.parse.unquote(s.split('/wiki/')[1]) for s in p['sources'] if 'fr.wikipedia.org/wiki/' in s] + [p['nom']]
if p['categorie'] == 'patrimoine' or p['type'] in ('village', 'ville'): titles += search(p['nom'] + ' Vercors')[:1]
seen = set()
print(i, p["nom"], flush=True)
for t in titles:
if t in seen: continue
seen.add(t); r = summ(t)
if not r or r.get('type') != 'standard' or not r.get('coordinates') or not r.get('extract'): continue
d = dist(p['lat'], p['lon'], r['coordinates']['lat'], r['coordinates']['lon'])
lim = 6 if p['type'] in ('village', 'ville') else 2
if d > lim: continue
ext = r['extract']
if p['type'] not in ('village','ville') and re.search(r'est une commune|est un sommet|est une ancienne commune', ext[:120]): continue
if 'peut désigner' in ext or len(ext) < 40: continue
# nom : au moins un mot significatif commun
w = [x for x in re.findall(r'\w{4,}', p['nom'].lower())]
if w and not any(x in (r['title'] + ' ' + ext[:200]).lower() for x in w): continue
cur[p['id']] = {'nom': p['nom'], 'complements': {'description': short(ext), 'complements_source': r['content_urls']['desktop']['page'] + ' (Wikipedia FR, CC BY-SA 4.0 ; correspondance validee par coordonnees, ecart %.1f km)' % d}}
ok += 1; break
if i % 10 == 0:
json.dump(cur, open(F + '.tmp', 'w', encoding='utf8'), ensure_ascii=False, indent=1); os.replace(F + '.tmp', F); print(i, ok, flush=True)
json.dump(cur, open(F + '.tmp', 'w', encoding='utf8'), ensure_ascii=False, indent=1); os.replace(F + '.tmp', F)
print('fini', ok)