Code for parsing all params, (just not the zipped yet).

Still the postgres requests to add (will be done)
Locally tested with various params, reading from CSV file.

porticapi/README.md

@@ -60,14 +60,26 @@ pointcalls : les données d'observation à chaque escale des navires
 Liste des paramètres communs à chaque requête et valeur par défaut en gras: 
 - params : **all** | tableau des noms longs des attributs de l'API à renvoyer
 - format : csv | **json**
-- zipped : true | **false**
 - shortenfields : true | **false**
 - both_to : true | **false**
+- zipped : true | **false**
+
 
 Si le parametre n'est pas précisé, le serveur opte pour la valeur de paramètre par défaut (en gras).
 
 URL : **http://data.portic.fr/api/**
 
+```py
+def formatCSV2_deprecated(data):
+    dest = io.StringIO()
+    dest.write('[toto, 1, tutu]')
+    output = make_response(dest.getvalue())
+    #output = excel.make_response_from_array(data, 'csv')
+    output.headers["Content-Disposition"] = "attachment; filename=export.csv"
+    output.headers["Content-type"] = "text/csv"
+    return output
+```
+
 ### 1.2.3. /pointcalls?
 -- TOUT en json
 

porticapi/apidata.py

-# -*- coding: utf-8 -*-
+#!/usr/bin/python3
+# -*-coding:UTF-8 -*
 '''
 Created on 14 may 2020
 @author: cplumejeaud, ggeoffroy
@@ -9,19 +10,27 @@ This requires :
 @see navigocorpus/ETL/BuildNavigoviz.py
 '''
 
-from flask import Flask, jsonify, abort, render_template,url_for,request
+from flask import Flask, jsonify, abort, render_template,url_for,request, make_response
 from flask_cors import CORS, cross_origin
+
 from flask_caching import Cache
 import numpy as np
 import csv
 import json
+import io
+
+#import flask_ext 
+#import flask_excel as excel
+#import pyexcel as pe
+
+
 
 app = Flask(__name__)
 CORS(app)
 
-app.config.from_object('config')
-port = app.config['PORT']      
-#port = '80'
+#app.config.from_object('config')
+#port = app.config['PORT']      
+port = '80'
 
 def isInside(lat, lon, radius, x, y):    
     """
@@ -44,16 +53,89 @@ def calcul_isInside(lat, lon, radius):
     calculated = df.apply(lambda row:isInside(lat,lon,radius/111.19492664455873,row.departure_latitude,row.departure_longitude), axis=1)
     return df.loc[calculated]
 
+
+
+def formatCSV(mydataframe):
+    #print(mydataframe)
+    #sheet = pe.Sheet(data)
+    #sheet.save_to_memory("csv", dest)
+
+    #Options de compression possibles to_csv 
+    #https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_csv.html
+    csvdata = mydataframe.to_csv()
+
+    #https://stackoverflow.com/questions/26997679/writing-a-csv-from-flask-framework
+    dest = io.StringIO()
+    dest.write(csvdata)
+    output = make_response(dest.getvalue())
+    output.headers["Content-Disposition"] = "attachment; filename=export.csv"
+    output.headers["Content-type"] = "text/csv"
+    return output
+
+def formatJSON(data):
+    json_str = json.dumps(data.to_json(orient='records'))
+    return json.loads(json_str)
+
+def formatOutput(dfcsv):
+    """
+    Apply various formatting on output by processing request parameters
+    - format : csv | **json**
+    - zipped : true | **false**
+    - shortenfields : true | **false**
+    """
+
+    import pandas as pd
+
+    ##Shorten names or not
+    shortenfields = request.args.get("shortenfields")
+    #print('shortenfields ? '+shortenfields)
+    if (shortenfields != None and shortenfields=='true') :
+        #API;colname;short_colname
+        mapnames = pd.read_csv('static/data/shorten_names.csv', sep = ';') 
+        #Filter according API
+        mapnames = mapnames[mapnames['API']=='pointcalls']
+        #print(mapnames['colname'])
+        mapper=mapnames.loc[:, ['colname', 'short_colname'] ].set_index('colname')['short_colname'].to_dict()
+        #print(mapper) 
+        dfcsv = dfcsv.rename(columns=mapper)
+
+    ## Format output
+    format = request.args.get("format")
+    if (format != None and format == 'csv') :
+        return formatCSV(dfcsv)
+    else: 
+        return formatJSON(dfcsv)
+
 @app.route('/api/pointcalls/')   
 def getPointcalls():
     """
     Return the pointcalls as specified in API
     Will be extracted from postgres, schema navigoviz, table pointcall (see navigocorpus/ETL)
+        - params : **all** | tableau des noms longs des attributs de l'API à renvoyer
+
+    http://127.0.0.1:5004/api/pointcalls/?params=pointcall,pointcall_uhgs_id&shortenfields=true
+    http://127.0.0.1:5004/api/pointcalls/?format=csv
+
     """
     import pandas as pd
+    path = '/var/www/html/data/'
     dfcsv = pd.read_csv('static/data/pointcalls_API_11mai2020.csv', sep = ';')
-    json_str = json.dumps(dfcsv.to_json(orient='records'))
-    return json.loads(json_str)
+    #https://stackoverflow.com/questions/24251219/pandas-read-csv-low-memory-and-dtype-options
+    #pd.read_csv('static/data/pointcalls_API_11mai2020.csv', dtype={"all_cargos": object, "pkid": int})
+    #print(dfcsv.columns)
+    
+    ## Filter the result according requested params
+    params = request.args.get("params")
+    #params=pointcall,pointcall_uhgs_id
+    if (params is not None and len(params)>0) :
+        #print('selecting some columns')
+        keepparams = str(params).split(',') 
+        #keepparams = ['pkid', 'pointcall', 'pointcall_uhgs_id']
+        dfcsv = dfcsv.loc[:, keepparams]
+
+    return formatOutput(dfcsv)
+
+
 
 @app.route('/api/travels/')   
 def getTravels():
@@ -61,12 +143,38 @@ def getTravels():
     Return the travels as specified in API
     Will be extracted from postgres, schema navigoviz, table built_travels (see navigocorpus/ETL), 
     but with a filter by default : only source_entry = from and both-from, to avoid duplicates
+        - params : **all** | tableau des noms longs des attributs de l'API à renvoyer
+        - both_to : true | **false**
+
+    http://127.0.0.1:5004/api/travels/?format=csv&both_to=false
+
     """
     import pandas as pd
+    path = '/var/www/html/data/'
     dfcsv = pd.read_csv('static/data/travels_API_11mai2020.csv', sep = ';')
-    # Filter todo
-    json_str = json.dumps(dfcsv.to_json(orient='records'))
-    return json.loads(json_str)
+    
+    ## Filter the result according requested params
+    params = request.args.get("params")
+    #print(params)
+    if (params != None) :
+        #params=pointcall,pointcall_uhgs_id
+        keepparams = params.split(',') 
+        #keepparams = ['pkid', 'pointcall', 'pointcall_uhgs_id']
+        dfcsv = dfcsv.loc[:, keepparams]
+
+
+    # Filter to remove duplicates (by default) - if both_to is given, then do not filter and travels will contain duplicates
+    getduplicates = request.args.get("both_to")
+    #print(getduplicates)
+    if getduplicates is None or getduplicates != 'true' :
+        #Filter travels
+        #print('filtering duplicates out of travels')
+        dfcsv = dfcsv[dfcsv['source_entry']!='both-to']
+    
+    return formatOutput(dfcsv)
+
+
+
 
 @app.route('/api/details/departures/', methods = ['GET'])   
 def getDeparturesDetails():
@@ -79,7 +187,13 @@ def getDeparturesDetails():
     lon = float(request.args.get("lon"))
     radius = int(request.args.get("radius"))
     df1 = calcul_isInside(lat, lon, radius)
-    return json.loads(json.dumps(df1[['departure','departure_uhgs_id','departure_latitude','departure_longitude']].drop_duplicates().to_json(orient='records')))
+
+    dfcsv = df1[['departure','departure_uhgs_id','departure_latitude','departure_longitude']].drop_duplicates()
+
+    return formatOutput(dfcsv)
+
+
+    #return json.loads(json.dumps(df1[['departure','departure_uhgs_id','departure_latitude','departure_longitude']].drop_duplicates().to_json(orient='records')))
 
 @app.route('/api/agg/departures/', methods = ['GET']) 
 def getDeparturesAgg():
@@ -93,7 +207,11 @@ def getDeparturesAgg():
     radius = int(request.args.get("radius"))
     df2 = calcul_isInside(lat, lon, radius).departure.value_counts().reset_index()
     df2.columns = ['departure', 'count']
-    return json.loads(json.dumps(df2.to_json(orient='records')))   
+
+    
+    return formatOutput(df2)
+
+    #return json.loads(json.dumps(df2.to_json(orient='records')))   
 
 @app.route('/api/agg/destinations/', methods = ['GET'])   
 def getDestinationsAgg():
@@ -108,7 +226,10 @@ def getDestinationsAgg():
     df3 = calcul_isInside(lat, lon, radius).destination_amiraute.value_counts().reset_index()
     df3.columns = ['label', 'value']
     df3['id'] = df3['label']
-    return json.loads(json.dumps(df3.to_json(orient='records')))            
+
+    return formatOutput(df3)
+
+    #return json.loads(json.dumps(df3.to_json(orient='records')))            
 
 
 

porticapi/porticapi.wsgi

+#!/usr/bin/python3
+# -*-coding:UTF-8 -*
+
+import sys
+import os, os.path
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '.')))
+from porticwebapi import apidata as application

porticapi/static/data/shorten_names.csv

+API;colname;short_colname
+pointcalls;pkid;p00
+pointcalls;pointcall;p01
+pointcalls;pointcall_uhgs_id;p02
+pointcalls;pointcall_latitude;p03
+pointcalls;pointcall_longitude;p04
+pointcalls;pointcall_admiralty;p05
+pointcalls;pointcall_province;p06
+pointcalls;pointcall_states;p07
+pointcalls;pointcall_status;p08
+pointcalls;pointcall_shiparea;p09
+pointcalls;pointcall_point;p10
+pointcalls;pointcall_out_date;p11
+pointcalls;pointcall_action;p12
+pointcalls;outdate_fixed;p13
+pointcalls;pointcall_in_date;p14
+pointcalls;indate_fixed;p15
+pointcalls;net_route_marker;p16
+pointcalls;pointcall_fullrank;p17
+pointcalls;date_fixed;p18
+pointcalls;navigo_status;p19
+pointcalls;pointcall_function;p20
+pointcalls;ship_name;s00
+pointcalls;ship_id;s01
+pointcalls;tonnage;s02
+pointcalls;tonnage_unit;s03
+pointcalls;flag;s04
+pointcalls;class;s05
+pointcalls;homeport;s06
+pointcalls;homeport_uhgs_id;s07
+pointcalls;homeport_latitude;s08
+pointcalls;homeport_longitude;s09
+pointcalls;homeport_admiralty;s10
+pointcalls;homeport_province;s11
+pointcalls;homeport_states;s12
+pointcalls;homeport_status;s13
+pointcalls;homeport_shiparea;s14
+pointcalls;homeport_point;s15
+pointcalls;source_doc_id;s16
+pointcalls;source_text;s17
+pointcalls;source_suite;s18
+pointcalls;source_component;s19
+pointcalls;source_number;s20
+pointcalls;source_other;s21
+pointcalls;source_main_port_uhgs_id;s22
+pointcalls;source_main_port_toponyme;s23
+pointcalls;captain_id;c00
+pointcalls;captain_name;c01
+pointcalls;birthplace;c02
+pointcalls;status;c03
+pointcalls;citizenship;c04
+pointcalls;commodity_purpose;c05
+pointcalls;commodity_id;c06
+pointcalls;quantity;c07
+pointcalls;quantity_u;c08
+pointcalls;commodity_standardized;c09
+pointcalls;commodity_permanent_coding;c10
+pointcalls;all_cargos;c11
+pointcalls;tax_concept;t00
+pointcalls;payment_date;t01
+pointcalls;q01;t02
+pointcalls;q01_u;t03
+pointcalls;q02;t04
+pointcalls;q02_u;t05
+pointcalls;q03;t06
+pointcalls;q03_u;t07
+pointcalls;all_taxes;t08
+pointcalls;ship_uncertainty;u00
+pointcalls;tonnage_uncertainty;u01
+pointcalls;flag_uncertainty;u02
+pointcalls;homeport_uncertainty;u03
+pointcalls;pointcall_uncertainity;u04
+pointcalls;captain_uncertainity;u05
+pointcalls;cargo_uncertainity;u06
+pointcalls;taxe_uncertainity;u07
+travels;id;t01
+travels;distance_homeport_dep;t02
+travels;distance_dep_dest;t03
+travels;"departure	";s01
+travels;departure_uhgs_id;s02
+travels;departure_latitude;s03
+travels;departure_longitude;s04
+travels;departure_admiralty;s05
+travels;departure_province;s06
+travels;departure_states;s07
+travels;departure_status;s08
+travels;departure_shiparea;s09
+travels;"departure_point	";s10
+travels;departure_out_date;s11
+travels;departure_action;s12
+travels;outdate_fixed;s13
+travels;departure_navstatus;s14
+travels;departure_function;s15
+travels;destination;d01
+travels;destination_uhgs_id;d02
+travels;destination_latitude;d03
+travels;destination_longitude;d04
+travels;destination_admiralty;d05
+travels;destination_province;d06
+travels;destination_states;d07
+travels;destination_status;d08
+travels;departure_shiparea;d09
+travels;destination_point;d10
+travels;destination_action;d11
+travels;destination_in_date;d12
+travels;indate_fixed;d13
+travels;destination_navstatus;d14
+travels;destination_function;d15
+travels;ship_name;t04
+travels;ship_id;t05
+travels;tonnage;t06
+travels;tonnage_unit;t07
+travels;flag;t08
+travels;class;t09
+travels;homeport;t10
+travels;homeport_uhgs_id;t11
+travels;homeport_latitude;t12
+travels;homeport_longitude;t13
+travels;homeport_amiraute;t14
+travels;homeport_province;t15
+travels;homeport_states;t16
+travels;homeport_status;t17
+travels;homeport_shiparea;t18
+travels;homeport_point;t19
+travels;source_entry;t20
+travels;source_doc_id;t21
+travels;source_text;t22
+travels;source_suite;t23
+travels;source_component;t24
+travels;source_number;t25
+travels;source_other;t26
+travels;source_main_port_uhgs_id;t27
+travels;source_main_port_toponyme;t28
+travels;captain_id;t29
+travels;captain_name;t30
+travels;birthplace;t31
+travels;status;t32
+travels;citizenship;t33
+travels;commodity_purpose;t34
+travels;commodity_id;t35
+travels;quantity;t36
+travels;quantity_u;t37
+travels;commodity_standardized;t38
+travels;commodity_permanent_coding;t39
+travels;all_cargos;t40
+travels;tax_concept;t41
+travels;payment_date;t42
+travels;q01;t43
+travels;q01_u;t44
+travels;q02;t45
+travels;q02_u;t46
+travels;q03;t47
+travels;q03_u;t48
+travels;all_taxes;t49
+travels;ship_uncertainty;u01
+travels;tonnage_uncertainty;u02
+travels;flag_uncertainty;u03
+travels;homeport_uncertainty;u04
+travels;departure_uncertainity;u05
+travels;destination_uncertainity;u06
+travels;captain_uncertainity;u07
+travels;travel_uncertainity;u08
+travels;cargo_uncertainity;u09
+travels;taxe_uncertainity;u10