import json,csv,re,math,unicodedata,datetime,collections,os def norm(s): s=unicodedata.normalize('NFD',s.lower()); s=''.join(c for c in s if unicodedata.category(c)!='Mn') s=re.sub(r"[^a-z0-9]+",' ',s).strip() s=re.sub(r'\bst\b','saint',s); s=re.sub(r'\bste\b','sainte',s) return s com=json.load(open('communes.json',encoding='utf-8')) bydep={} for c in com: bydep.setdefault(c['codeDepartement'],{}).setdefault(norm(c['nom']),[]).append(c) def hav(a,b,c,d): R=6371;p=math.pi/180 x=math.sin((c-a)*p/2)**2+math.cos(a*p)*math.cos(c*p)*math.sin((d-b)*p/2)**2 return 2*R*math.asin(math.sqrt(x)) st=json.load(open('stage1.json',encoding='utf-8')) gf=json.load(open('geo_fr.json',encoding='utf-8')); gx=json.load(open('geo_foreign.json',encoding='utf-8')) def commune(dep,name): c=bydep[dep][norm(name)][0]; return c['nom'],c['centre']['coordinates'][1],c['centre']['coordinates'][0] OVR={'749':('73','Montvalezan'),'750':('73','Montvalezan'),'753':('73','Peisey-Nancroix'),'752':('73','Peisey-Nancroix'), '282':('73','Beaufort'),'33':('38','Le Haut-Bréda'),'186':('38','Plateau-des-Petites-Roches'),'746':('73','Saint-Genix-les-Villages'),'755':('73','Montvalezan')} for s in st: i=s['id_romma'] if i in OVR: d,nm=OVR[i] try: c,la,lo=commune(d,nm); s.update(commune=c,lat=la,lon=lo,departement=d,precision_coord='centre de commune ('+c+'), lieu exact inconnu (station de ski/lieu-dit), erreur possible de plusieurs km') except KeyError: print('ovr fail',i,nm) elif s['lat'] is None and s['pays']=='FR' and gf.get(i): la,lo,city,dc,idx,topo,v=gf[i] s.update(lat=la,lon=lo,commune=city or topo,departement=dc,precision_coord=('toponyme IGN "%s" (geocodage Geoplateforme), approx. 0,3 a 2 km'%topo) if idx=='poi' else 'centre de commune (IGN)') elif s['pays']!='FR' and gx.get(i): la,lo,dn=gx[i]; s.update(lat=la,lon=lo,commune=dn.split(',')[0],precision_coord='centre de la localite (Nominatim/OSM), approx.') elif i=='63': s.update(lat=45.6808,lon=6.8845,commune='Col du Petit-Saint-Bernard',precision_coord='saisi a la main (col), approx.') if i=='133': s.update(lat=45.095,lon=5.6,commune='Lans-en-Vercors',precision_coord='coordonnees de la station deja connue dans Weather Studio (fiche 000GS)') if s['lat'] is None: s['precision_coord']=None ic=[] for r in csv.DictReader(open('C:/Users/theog/Documents/_CODE/1-logiciels/weather-studio/GWS-Principal/data/stations_fr/stations_fr.csv',encoding='utf-8-sig')): if r['id'].endswith('GS'): continue try: ic.append(dict(src='infoclimat',id=r['id'],nom=r['nom'],lat=float(r['latitude']),lon=float(r['longitude']),alt=float(r['altitude_m']),dep=r['departement'].zfill(2),n=norm(r['nom']))) except Exception: pass mf=[] for r in csv.DictReader(open('mf.csv',encoding='utf-8')): if r['is_open']!='true' or r['end_date']: continue try: mf.append(dict(src='meteofrance',id=r['id'],nom=r['name'],lat=float(r['lat']),lon=float(r['lon']),alt=float(r['alt']),dep=r['department_id'].zfill(2),n=norm(r['name']+' '+(r['long_name'] or '')))) except Exception: pass refs=ic+mf def wb(needle,hay): return re.search(r'(^| )'+re.escape(needle)+r'( |$)',hay) is not None for s in st: nr=norm(s['nom_source']); main=norm(re.split(r'\s+-\s+|/',s['nom_source'])[0]) matches=[];near=[] for r in refs: da=abs(r['alt']-s['altitude_m']) d=hav(s['lat'],s['lon'],r['lat'],r['lon']) if s['lat'] is not None else None nm=(wb(nr,r['n']) or (len(main)>=4 and wb(main,r['n']))) and da<=100 if nm and (d is None or d<=25) and (d is not None or r['dep']==s['departement']): matches.append(dict(exact=(r['n'] in (nr,main)),source=r['src'],id=r['id'],nom=r['nom'],dist_km=None if d is None else round(d,2),delta_alt_m=round(da))) elif d is not None and d<=3 and da<=100: near.append(dict(source=r['src'],id=r['id'],nom=r['nom'],dist_km=round(d,2),delta_alt_m=round(da))) s['correspondances']=sorted(matches,key=lambda m:(m['dist_km'] is None,m['dist_km'] or 0))[:6] or sorted(near,key=lambda m:m['dist_km'])[:6] if matches: good=[m for m in matches if m['dist_km'] is not None and m['dist_km']<=5 and (m['exact'] or m['delta_alt_m']<=50)] if good: srcs={m['source'] for m in good} s['statut']='doublon-infoclimat' if 'infoclimat' in srcs else 'doublon-meteofrance' s['raison']='meme nom/commune et altitude a +-100 m (%s)'%good[0]['nom'] elif s['lat'] is None: s['statut']='a-verifier'; s['raison']='nom correspondant a une station existante, mais pas de coordonnees Romma pour confirmer' else: s['statut']='a-verifier'; s['raison']='nom proche d une station existante mais nom different/altitude ecart 50-100 m/ou >5 km : a confirmer (coordonnees Romma approximatives)' elif near: s['statut']='a-verifier'; s['raison']='station Infoclimat/Meteo-France a <3 km et altitude +-100 m, nom different (coordonnees Romma approximatives)' elif s['lat'] is None: s['statut']='a-verifier'; s['raison']='coordonnees introuvables, comparaison impossible' else: s['statut']='unique'; s['raison']='aucune station Infoclimat/Meteo-France a <3 km et +-100 m, ni de meme nom' s['nom']=s['nom_source'].title() if s['nom_source'].isupper() else s['nom_source'] s['url_station_romma']='https://www.romma.fr/station_24.php?id='+s['id_romma'] s['source_info']='romma.fr/nos_stations.php, menu public (id, nom, altitude, departement) ; coordonnees: '+(s['precision_coord'] or 'non determinees')+'. Page station non consultee (robots.txt).' out=dict(genere=datetime.date.today().isoformat(), criteres='doublon = meme nom/commune ET altitude +-100 m ET station a <=5 km ; a-verifier = station a <3 km (+-100 m) de nom different, ou comparaison impossible ; sinon unique. Coordonnees Romma approximatives (non publiques).', sources=['https://www.romma.fr/nos_stations.php','Infoclimat StatIC: stations_fr.csv de Weather Studio (opendata/stations_xhr.php, %d stations)'%len(ic),'Meteo-France: data.gouv.fr, Informations sur les stations Meteo-France (metadonnees), %d stations ouvertes'%len(mf),'IGN Geoplateforme geocodage; geo.api.gouv.fr communes; Nominatim (CH/IT)'], stations=st) D='C:/Users/theog/Documents/_CODE/6-sites/stations-romma' os.makedirs(D+'/data',exist_ok=True) json.dump(out,open(D+'/data/stations-romma.json','w',encoding='utf-8'),ensure_ascii=False,indent=1) open(D+'/data/stations-romma.js','w',encoding='utf-8').write('window.STATIONS_ROMMA='+json.dumps(out,ensure_ascii=False)+';') print(collections.Counter(s['statut'] for s in st), 'sans coord',sum(1 for s in st if s['lat'] is None)) for s in st: if s['statut'].startswith('doublon') or s['id_romma'] in('133','762'): print(s['id_romma'],s['nom_source'],s['altitude_m'],s['statut'],s['correspondances'][:1])