Commit 74ac1aac authored by Bernard PRADINES's avatar Bernard PRADINES
Browse files

Ajout de /api/sources avec sa doc

parent 8f471af5
__pycache__/*
.venv/*
_requirements.txt
a_installer_pip.txt
gethelp.bat
help.txt
README.pdf
\ No newline at end of file
README.pdf
test.py
\ No newline at end of file
This diff is collapsed.
......@@ -20,6 +20,7 @@ import json
import io
import os
import psycopg2 as pg
import re
#import flask_ext
#import flask_excel as excel
......@@ -660,6 +661,55 @@ def getFlows():
return formatOutput(dfcsv, 'travels')
def validateSrid(srid):
"""
Internal method
Checks in spatial_ref_sys of the database if given srid is valid, otherwise returns 900913
"""
if srid is None :
srid = '900913'
else :
#we chek if it is valid by looking in the postgres spatial_ref_sys table : 4326, 3857 for instance
query = """select distinct srid from public.spatial_ref_sys srs"""
srids = retrieveDataFromPostgres(query)
if int(srid) not in srids['srid'].tolist() :
srid = '900913'
return srid
def getFilterdateForPointcall(date):
"""
Internal method
returns SQL to filter pointcalls by date (4 digits format)
"""
## filter following a date
filterDate = 'where true'
if (date is not None and len(date)==4) :
filterDate = """ where (substring(pointcall_out_date for 4) = '%s' or substring(pointcall_in_date for 4) = '%s') """ % (date, date)
return filterDate
def validateLang(lang):
"""
Internal method
returns "fr" or "en" (defaults to "fr" if no lang or anything else is given)
"""
result="fr" # par défaut
if lang is not None:
lang_en_minuscules=lang.lower()
# tests if lang_en_minuscules matches a regular expression :
regexp="^(fr|en)$" # ^ = at the start of string, $ = at the end of string => exactly equal to 'fr' or (|) 'en'
match=re.match(regexp, lang_en_minuscules) # tests if lang, converted to lowercase, is exactly "fr" or "en"
if match is not None:
result=match[0]
return result
def getLocalizedFieldnamesForSources(lang):
"""
Internal method
returns a string containing the correct names of the fields in the database (comma-separated) for "fr" or "en" languages
"""
return f"toponyme_standard_{lang} as toponym, substate_1789_{lang} as substate_1789,"
@app.route('/api/ports/')
def getPorts():
"""
......@@ -680,25 +730,12 @@ def getPorts():
"""
# select the srid given by user for geometry transformation
srid = request.args.get("srid")
#print (srid)
if srid is None :
srid = '900913'
else :
#we chek if it is valid by looking in the postgres spatial_ref_sys table : 4326, 3857 for instance
query = """select distinct srid from public.spatial_ref_sys srs"""
srids = retrieveDataFromPostgres(query)
if int(srid) not in srids['srid'].tolist() :
srid = '900913'
print (srid)
srid=validateSrid(srid)
## filter following a date
filterDate = 'where true'
# filter following a date
date = request.args.get("date")
if (date is not None and len(date)==4) :
filterDate = """ where (substring(pointcall_out_date for 4) = '%s' or substring(pointcall_in_date for 4) = '%s') """ % (date, date)
filterDate=getFilterdateForPointcall(date)
query = """SELECT ogc_fid, uhgs_id, total, toponyme as toponym, belonging_states, belonging_substates, status, geonameid, amiraute as admiralty, province, shiparea , ST_AsGeoJSON(ST_Transform(geom, %s)) as point
FROM ports.port_points p,
(select pointcall_uhgs_id, count( *) as total
......@@ -708,6 +745,128 @@ def getPorts():
data = retrieveDataFromPostgres(query)
return formatOutput(data, 'ports')
def getTupleNomChamp(champ):
"""
Internal method
called by handleParams(champs) :
returns a tuple (nom='name of SQL field', champ='code of field in select SQL clause')
"""
separateur=" as "
if separateur in champ :
parties_champ=champ.split(separateur)
nom=parties_champ[1].strip()
else :
nom=champ.strip()
return (nom, champ.strip())
def handleParams(champs):
"""
internal method
if the 'params' parameter is defined, this function filters the 'champs' parameter (string) to keep only desired ones
'champs' is the content of the SQL select clause : fields are separated by ", " (and ", " is nowhere else in the clause !), and a field can be renamed with " as "
"""
params=request.args.get("params")
if (params is None) or (params.lower().strip() == "all"): # on ne souhaite pas filtrer les champs renvoyés
pass # renvoyer directement le contenu de la clause select complète
else:
# faire un dictionnaire des champs de la clause select
list_champs=champs.split(", ")
dict_champs=dict([getTupleNomChamp(champ) for champ in list_champs]) # dictionnaire {"nom du champ": "code du champ"}
# séparer les champs demandés :
list_params=params.replace(" ", "").split(",") # retire les espaces de 'params', et sépare les paramètres demandés avec ","
# filtrer le dictionnaire des champs, pour ne garder que ceux demandés :
list_champs_gardes=[dict_champs[nom_champ] for nom_champ in list_params if nom_champ in dict_champs]
# assembler le contenu, prêt pour la clause 'select' :
champs=", ".join(list_champs_gardes)
return champs
@app.route('/api/sources/')
def getSources():
"""
Get statistics about the amount of pointcall data found in sources (couverture des sources de navigo), related to ports.
In a given source (source_suite = G5 | Marseille | ...), lists the ports (uhgs_id) of France (state_1789_fr = 'France'), and for each port, counts :
- total : total number of observed ("O") pointcalls found for this port in this source, at a given date in time
- nb_tonnage_filled : number of pointcalls where tonnage data (volume de marchandise en tonneaux ou quintaux) is given
- nb_homeport_filled : number of pointcalls where homeport (port d'attache du navire) is known
- nb_product_filled : number of pointcalls where commodity_purpose (nature de la marchandise transportée, ou objet du voyage) is given
- nb_birthplace_filled : number of pointcalls where birthplace (lieu d'origine du capitaine du bateau) is known
- nb_citizenship_filled : number of pointcalls where citizenship (nationalité du capitaine du bateau) is known
- good_sum_tonnage : sum of the tonnage that goes by this port (converted to barrel unit if given in quintals)
A join is made between pointcall and port data, to get nearly the same information about a port as with the ports api (see /api/fieldnames?api=ports) :
- ogc_fid : id
- uhgs_id : geo_general text id of the port
- toponym : standardised name of the port, in the 'lang' language
- substate_1789 : belonging substate of the port in 1787 / 1789. In the 'lang' language
- status : null | "oblique" | "siège amirauté", type of port
- geonameid : nearest geoname identifier for the port
- admiralty : name of the home admiralty (amirauté) for the port, in 1787
- province : name of the home province for the port, in 1787
- shiparea : name of the maritime area for the port in 2020
- point : coordinates for representation on a map
Will be extracted from postgres, schema navigoviz, table pointcall (see navigocorpus/ETL), completed with schema ports, table port_points.
Parameters :
- srid : **900913** | 4326 | 3857, for geometry transformation of the coordinates of the point representing the port on a map
- date : **none** | 4 digits representing a year (1787 | 1789), to extract pointcall data from the source, for this given year only
- lang : **fr** | en, language for toponym(_standard), substate_1789
- params : **all** | coma-separated list of fields to be returned by the API
Examples :
- http://localhost/api/sources/?srid=4326
- http://localhost/api/sources/?srid=4326&date=1789
- http://localhost/api/sources/?srid=4326&date=1789&lang=en
- http://localhost/api/sources/?srid=4326&params=uhgs_id,toponym,total,point
"""
# select the srid given by user for geometry transformation
srid = request.args.get("srid")
srid=validateSrid(srid)
# filter following a date
date = request.args.get("date")
filterDate=getFilterdateForPointcall(date)
# language given by user (defaults to "fr")
lang = request.args.get("lang") # read requested language
lang=validateLang(lang) # "fr" or "en" valid only (defaults to "fr")
localizedDatabaseFieldNames=getLocalizedFieldnamesForSources(lang)
# liste de tous les champs qui peuvent être renvoyés, formatés pour la clause 'select' :
fields_for_select=f"""
ogc_fid, uhgs_id, {localizedDatabaseFieldNames} status, geonameid, amiraute as admiralty, province, shiparea , ST_AsGeoJSON(ST_Transform(geom,{srid})) as point,
source_suite, total, nb_tonnage_filled, nb_homeport_filled, nb_product_filled, nb_birthplace_filled, nb_citizenship_filled, good_sum_tonnage
"""
fields_for_select=handleParams(fields_for_select) # ne garde que les champs demandés dans 'params'
# assemble the SQL query :
query=f"""
select {fields_for_select}
from ports.port_points pp ,
(
select pointcall_uhgs_id, source_suite,
count(pkid) as total ,
count(tonnage) as nb_tonnage_filled,
count(homeport) as nb_homeport_filled,
count(commodity_purpose) as nb_product_filled,
count(birthplace) as nb_birthplace_filled,
count(citizenship) as nb_citizenship_filled,
sum(tonnage_numeric) as good_sum_tonnage
from (
select source_suite, pointcall_uhgs_id, pkid, tonnage, homeport, commodity_purpose, birthplace, citizenship,
(case when tonnage_unit = 'quintaux' then (tonnage::float)/24 else tonnage::float end) as tonnage_numeric
from navigoviz.pointcall p {filterDate} and state_1789_fr = 'France' and pointcall_function = 'O'
) as k
group by pointcall_uhgs_id , source_suite
) as toto
where toto.pointcall_uhgs_id = pp.uhgs_id
"""
# print(query)
# execute SQL query, and format resulting data :
data = retrieveDataFromPostgres(query)
return formatOutput(data, 'ports')
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment