Commit df96dd9a authored by Christine Plumejeaud's avatar Christine Plumejeaud
Browse files

Code for parsing all params, (just not the zipped yet).

Still the postgres requests to add (will be done)
Locally tested with various params, reading from CSV file.
parent e642548e
......@@ -60,14 +60,26 @@ pointcalls : les données d'observation à chaque escale des navires
Liste des paramètres communs à chaque requête et valeur par défaut en gras:
- params : **all** | tableau des noms longs des attributs de l'API à renvoyer
- format : csv | **json**
- zipped : true | **false**
- shortenfields : true | **false**
- both_to : true | **false**
- zipped : true | **false**
Si le parametre n'est pas précisé, le serveur opte pour la valeur de paramètre par défaut (en gras).
URL : **http://data.portic.fr/api/**
```py
def formatCSV2_deprecated(data):
dest = io.StringIO()
dest.write('[toto, 1, tutu]')
output = make_response(dest.getvalue())
#output = excel.make_response_from_array(data, 'csv')
output.headers["Content-Disposition"] = "attachment; filename=export.csv"
output.headers["Content-type"] = "text/csv"
return output
```
### 1.2.3. /pointcalls?
-- TOUT en json
......
# -*- coding: utf-8 -*-
#!/usr/bin/python3
# -*-coding:UTF-8 -*
'''
Created on 14 may 2020
@author: cplumejeaud, ggeoffroy
......@@ -9,19 +10,27 @@ This requires :
@see navigocorpus/ETL/BuildNavigoviz.py
'''
from flask import Flask, jsonify, abort, render_template,url_for,request
from flask import Flask, jsonify, abort, render_template,url_for,request, make_response
from flask_cors import CORS, cross_origin
from flask_caching import Cache
import numpy as np
import csv
import json
import io
#import flask_ext
#import flask_excel as excel
#import pyexcel as pe
app = Flask(__name__)
CORS(app)
app.config.from_object('config')
port = app.config['PORT']
#port = '80'
#app.config.from_object('config')
#port = app.config['PORT']
port = '80'
def isInside(lat, lon, radius, x, y):
"""
......@@ -44,16 +53,89 @@ def calcul_isInside(lat, lon, radius):
calculated = df.apply(lambda row:isInside(lat,lon,radius/111.19492664455873,row.departure_latitude,row.departure_longitude), axis=1)
return df.loc[calculated]
def formatCSV(mydataframe):
#print(mydataframe)
#sheet = pe.Sheet(data)
#sheet.save_to_memory("csv", dest)
#Options de compression possibles to_csv
#https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_csv.html
csvdata = mydataframe.to_csv()
#https://stackoverflow.com/questions/26997679/writing-a-csv-from-flask-framework
dest = io.StringIO()
dest.write(csvdata)
output = make_response(dest.getvalue())
output.headers["Content-Disposition"] = "attachment; filename=export.csv"
output.headers["Content-type"] = "text/csv"
return output
def formatJSON(data):
json_str = json.dumps(data.to_json(orient='records'))
return json.loads(json_str)
def formatOutput(dfcsv):
"""
Apply various formatting on output by processing request parameters
- format : csv | **json**
- zipped : true | **false**
- shortenfields : true | **false**
"""
import pandas as pd
##Shorten names or not
shortenfields = request.args.get("shortenfields")
#print('shortenfields ? '+shortenfields)
if (shortenfields != None and shortenfields=='true') :
#API;colname;short_colname
mapnames = pd.read_csv('static/data/shorten_names.csv', sep = ';')
#Filter according API
mapnames = mapnames[mapnames['API']=='pointcalls']
#print(mapnames['colname'])
mapper=mapnames.loc[:, ['colname', 'short_colname'] ].set_index('colname')['short_colname'].to_dict()
#print(mapper)
dfcsv = dfcsv.rename(columns=mapper)
## Format output
format = request.args.get("format")
if (format != None and format == 'csv') :
return formatCSV(dfcsv)
else:
return formatJSON(dfcsv)
@app.route('/api/pointcalls/')
def getPointcalls():
"""
Return the pointcalls as specified in API
Will be extracted from postgres, schema navigoviz, table pointcall (see navigocorpus/ETL)
- params : **all** | tableau des noms longs des attributs de l'API à renvoyer
http://127.0.0.1:5004/api/pointcalls/?params=pointcall,pointcall_uhgs_id&shortenfields=true
http://127.0.0.1:5004/api/pointcalls/?format=csv
"""
import pandas as pd
path = '/var/www/html/data/'
dfcsv = pd.read_csv('static/data/pointcalls_API_11mai2020.csv', sep = ';')
json_str = json.dumps(dfcsv.to_json(orient='records'))
return json.loads(json_str)
#https://stackoverflow.com/questions/24251219/pandas-read-csv-low-memory-and-dtype-options
#pd.read_csv('static/data/pointcalls_API_11mai2020.csv', dtype={"all_cargos": object, "pkid": int})
#print(dfcsv.columns)
## Filter the result according requested params
params = request.args.get("params")
#params=pointcall,pointcall_uhgs_id
if (params is not None and len(params)>0) :
#print('selecting some columns')
keepparams = str(params).split(',')
#keepparams = ['pkid', 'pointcall', 'pointcall_uhgs_id']
dfcsv = dfcsv.loc[:, keepparams]
return formatOutput(dfcsv)
@app.route('/api/travels/')
def getTravels():
......@@ -61,12 +143,38 @@ def getTravels():
Return the travels as specified in API
Will be extracted from postgres, schema navigoviz, table built_travels (see navigocorpus/ETL),
but with a filter by default : only source_entry = from and both-from, to avoid duplicates
- params : **all** | tableau des noms longs des attributs de l'API à renvoyer
- both_to : true | **false**
http://127.0.0.1:5004/api/travels/?format=csv&both_to=false
"""
import pandas as pd
path = '/var/www/html/data/'
dfcsv = pd.read_csv('static/data/travels_API_11mai2020.csv', sep = ';')
# Filter todo
json_str = json.dumps(dfcsv.to_json(orient='records'))
return json.loads(json_str)
## Filter the result according requested params
params = request.args.get("params")
#print(params)
if (params != None) :
#params=pointcall,pointcall_uhgs_id
keepparams = params.split(',')
#keepparams = ['pkid', 'pointcall', 'pointcall_uhgs_id']
dfcsv = dfcsv.loc[:, keepparams]
# Filter to remove duplicates (by default) - if both_to is given, then do not filter and travels will contain duplicates
getduplicates = request.args.get("both_to")
#print(getduplicates)
if getduplicates is None or getduplicates != 'true' :
#Filter travels
#print('filtering duplicates out of travels')
dfcsv = dfcsv[dfcsv['source_entry']!='both-to']
return formatOutput(dfcsv)
@app.route('/api/details/departures/', methods = ['GET'])
def getDeparturesDetails():
......@@ -79,7 +187,13 @@ def getDeparturesDetails():
lon = float(request.args.get("lon"))
radius = int(request.args.get("radius"))
df1 = calcul_isInside(lat, lon, radius)
return json.loads(json.dumps(df1[['departure','departure_uhgs_id','departure_latitude','departure_longitude']].drop_duplicates().to_json(orient='records')))
dfcsv = df1[['departure','departure_uhgs_id','departure_latitude','departure_longitude']].drop_duplicates()
return formatOutput(dfcsv)
#return json.loads(json.dumps(df1[['departure','departure_uhgs_id','departure_latitude','departure_longitude']].drop_duplicates().to_json(orient='records')))
@app.route('/api/agg/departures/', methods = ['GET'])
def getDeparturesAgg():
......@@ -93,7 +207,11 @@ def getDeparturesAgg():
radius = int(request.args.get("radius"))
df2 = calcul_isInside(lat, lon, radius).departure.value_counts().reset_index()
df2.columns = ['departure', 'count']
return json.loads(json.dumps(df2.to_json(orient='records')))
return formatOutput(df2)
#return json.loads(json.dumps(df2.to_json(orient='records')))
@app.route('/api/agg/destinations/', methods = ['GET'])
def getDestinationsAgg():
......@@ -108,7 +226,10 @@ def getDestinationsAgg():
df3 = calcul_isInside(lat, lon, radius).destination_amiraute.value_counts().reset_index()
df3.columns = ['label', 'value']
df3['id'] = df3['label']
return json.loads(json.dumps(df3.to_json(orient='records')))
return formatOutput(df3)
#return json.loads(json.dumps(df3.to_json(orient='records')))
......
#!/usr/bin/python3
# -*-coding:UTF-8 -*
import sys
import os, os.path
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '.')))
from porticwebapi import apidata as application
API;colname;short_colname
pointcalls;pkid;p00
pointcalls;pointcall;p01
pointcalls;pointcall_uhgs_id;p02
pointcalls;pointcall_latitude;p03
pointcalls;pointcall_longitude;p04
pointcalls;pointcall_admiralty;p05
pointcalls;pointcall_province;p06
pointcalls;pointcall_states;p07
pointcalls;pointcall_status;p08
pointcalls;pointcall_shiparea;p09
pointcalls;pointcall_point;p10
pointcalls;pointcall_out_date;p11
pointcalls;pointcall_action;p12
pointcalls;outdate_fixed;p13
pointcalls;pointcall_in_date;p14
pointcalls;indate_fixed;p15
pointcalls;net_route_marker;p16
pointcalls;pointcall_fullrank;p17
pointcalls;date_fixed;p18
pointcalls;navigo_status;p19
pointcalls;pointcall_function;p20
pointcalls;ship_name;s00
pointcalls;ship_id;s01
pointcalls;tonnage;s02
pointcalls;tonnage_unit;s03
pointcalls;flag;s04
pointcalls;class;s05
pointcalls;homeport;s06
pointcalls;homeport_uhgs_id;s07
pointcalls;homeport_latitude;s08
pointcalls;homeport_longitude;s09
pointcalls;homeport_admiralty;s10
pointcalls;homeport_province;s11
pointcalls;homeport_states;s12
pointcalls;homeport_status;s13
pointcalls;homeport_shiparea;s14
pointcalls;homeport_point;s15
pointcalls;source_doc_id;s16
pointcalls;source_text;s17
pointcalls;source_suite;s18
pointcalls;source_component;s19
pointcalls;source_number;s20
pointcalls;source_other;s21
pointcalls;source_main_port_uhgs_id;s22
pointcalls;source_main_port_toponyme;s23
pointcalls;captain_id;c00
pointcalls;captain_name;c01
pointcalls;birthplace;c02
pointcalls;status;c03
pointcalls;citizenship;c04
pointcalls;commodity_purpose;c05
pointcalls;commodity_id;c06
pointcalls;quantity;c07
pointcalls;quantity_u;c08
pointcalls;commodity_standardized;c09
pointcalls;commodity_permanent_coding;c10
pointcalls;all_cargos;c11
pointcalls;tax_concept;t00
pointcalls;payment_date;t01
pointcalls;q01;t02
pointcalls;q01_u;t03
pointcalls;q02;t04
pointcalls;q02_u;t05
pointcalls;q03;t06
pointcalls;q03_u;t07
pointcalls;all_taxes;t08
pointcalls;ship_uncertainty;u00
pointcalls;tonnage_uncertainty;u01
pointcalls;flag_uncertainty;u02
pointcalls;homeport_uncertainty;u03
pointcalls;pointcall_uncertainity;u04
pointcalls;captain_uncertainity;u05
pointcalls;cargo_uncertainity;u06
pointcalls;taxe_uncertainity;u07
travels;id;t01
travels;distance_homeport_dep;t02
travels;distance_dep_dest;t03
travels;"departure ";s01
travels;departure_uhgs_id;s02
travels;departure_latitude;s03
travels;departure_longitude;s04
travels;departure_admiralty;s05
travels;departure_province;s06
travels;departure_states;s07
travels;departure_status;s08
travels;departure_shiparea;s09
travels;"departure_point ";s10
travels;departure_out_date;s11
travels;departure_action;s12
travels;outdate_fixed;s13
travels;departure_navstatus;s14
travels;departure_function;s15
travels;destination;d01
travels;destination_uhgs_id;d02
travels;destination_latitude;d03
travels;destination_longitude;d04
travels;destination_admiralty;d05
travels;destination_province;d06
travels;destination_states;d07
travels;destination_status;d08
travels;departure_shiparea;d09
travels;destination_point;d10
travels;destination_action;d11
travels;destination_in_date;d12
travels;indate_fixed;d13
travels;destination_navstatus;d14
travels;destination_function;d15
travels;ship_name;t04
travels;ship_id;t05
travels;tonnage;t06
travels;tonnage_unit;t07
travels;flag;t08
travels;class;t09
travels;homeport;t10
travels;homeport_uhgs_id;t11
travels;homeport_latitude;t12
travels;homeport_longitude;t13
travels;homeport_amiraute;t14
travels;homeport_province;t15
travels;homeport_states;t16
travels;homeport_status;t17
travels;homeport_shiparea;t18
travels;homeport_point;t19
travels;source_entry;t20
travels;source_doc_id;t21
travels;source_text;t22
travels;source_suite;t23
travels;source_component;t24
travels;source_number;t25
travels;source_other;t26
travels;source_main_port_uhgs_id;t27
travels;source_main_port_toponyme;t28
travels;captain_id;t29
travels;captain_name;t30
travels;birthplace;t31
travels;status;t32
travels;citizenship;t33
travels;commodity_purpose;t34
travels;commodity_id;t35
travels;quantity;t36
travels;quantity_u;t37
travels;commodity_standardized;t38
travels;commodity_permanent_coding;t39
travels;all_cargos;t40
travels;tax_concept;t41
travels;payment_date;t42
travels;q01;t43
travels;q01_u;t44
travels;q02;t45
travels;q02_u;t46
travels;q03;t47
travels;q03_u;t48
travels;all_taxes;t49
travels;ship_uncertainty;u01
travels;tonnage_uncertainty;u02
travels;flag_uncertainty;u03
travels;homeport_uncertainty;u04
travels;departure_uncertainity;u05
travels;destination_uncertainity;u06
travels;captain_uncertainity;u07
travels;travel_uncertainity;u08
travels;cargo_uncertainity;u09
travels;taxe_uncertainity;u10
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment