Commit 22949053 authored by Christine Plumejeaud's avatar Christine Plumejeaud
Browse files

fix date format

parent 66e81836
......@@ -72,7 +72,8 @@ def retrieveDataFromPostgres(query, cast_columns_to_types=None) :
Internal method to select data using SQL query
return a dataframe
see read_sql_query() doc : https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_sql_query.html
columns can be type-casted using the cast_columns_to_types dictionary argument : { 'column_sql_name' : 'type'}. For 'type', see dtype String Aliases : https://pandas.pydata.org/docs/user_guide/basics.html#basics-dtypes
columns can be type-casted using the cast_columns_to_types dictionary argument : { 'column_sql_name' : 'type'}.
For 'type', see dtype String Aliases : https://pandas.pydata.org/docs/user_guide/basics.html#basics-dtypes
"""
#import pandas.io.sql as psql
import pandas as pd
......@@ -105,8 +106,10 @@ def formatCSV(mydataframe):
def formatJSON(dataframe):
"""
Internal method to output dataframe as JSON
https://pandas.pydata.org/pandas-docs/version/0.23/generated/pandas.DataFrame.to_json.html : says that date may be sent like epoch in ms
Added date_format='iso' to avoid this.
"""
json_str = json.dumps(dataframe.to_json(orient='records'))
json_str = json.dumps(dataframe.to_json(orient='records', date_format='iso'))
return json.loads(json_str)
def df_to_geojson(df, properties, id = 'ogc_fid', lat='y', lon='x'):
......@@ -1149,8 +1152,20 @@ def getShipsDescription():
occurences_class, occurences_tonnageclass, captain_list, mindate, maxdate, nb_sourcedoc
from navigoviz.ship"""
## Warning, no comments have been set on ship table
#I've tried this, and it was working. Yet dates had dummy time zone inside (T00:00:00.000Z), e.g. 1787-01-02T00:00:00.000Z
#Thus, I've changed the type of mindate and maxdate to text fields in database (yyyy-mm-dd format)
cast_df_columns_to_types={
'ship_id' : 'string',
'occurences_names' : 'string',
'occurences_homeports' : 'string',
'occurences_flags' : 'string',
'occurences_class' : 'string',
'occurences_tonnageclass' :'string',
'captain_list':'string',
'mindate': 'datetime64[ns]',
'maxdate' : 'datetime64[ns]',
'nb_sourcedoc': 'Int16'
}
## Si jamais on demande shortenFields= true, ca va planter.
return retrieveDataAndFormatOutput(query, api='ships')
......@@ -1178,9 +1193,7 @@ def getCaptainsDescription():
query = f"""select captain_id, occurences_names, {localizedDatabaseFieldNames},
ship_list, mindate, maxdate, nb_sourcedoc
from navigoviz.captain"""
##Warning, no comments have been set on captain table
## Si jamais on demande shortenFields= true, ca va planter.
## Si jamais on demande shortenFields= true, ca va planter.
return retrieveDataAndFormatOutput(query, api='captains')
@app.route('/api/test/')
......
......@@ -275,6 +275,7 @@ order by travel_rank;
-- Renvoyer les variantes de noms de navires ou de capitaines
-- role api_user / password portic
GRANT USAGE ON SCHEMA navigoviz TO api_user;
GRANT USAGE ON SCHEMA navigocheck to api_user;
GRANT USAGE ON SCHEMA navigo to api_user;
......@@ -331,6 +332,16 @@ alter table navigoviz.captain add column mindate date;
alter table navigoviz.captain add column maxdate date;
alter table navigoviz.captain add column nb_sourcedoc int;
alter table navigoviz.ship alter column mindate type text;
alter table navigoviz.ship alter column maxdate type text;
-- alter table navigoviz.ship alter column mindate type date using mindate::date;
-- alter table navigoviz.ship alter column maxdate type date using maxdate::date;
-- alter table navigoviz.captain alter column mindate type date using mindate::date;
-- alter table navigoviz.captain alter column maxdate type date using maxdate::date;
alter table navigoviz.captain alter column mindate type text;
alter table navigoviz.captain alter column maxdate type text;
comment on column navigoviz.ship.captain_list is 'JSON array of various captains having sailed on the same ship through sources, with the frequecy of appearance in documents';
comment on column navigoviz.ship.mindate is 'First date of appearance of this ship in the sources';
comment on column navigoviz.ship.maxdate is 'Last date of appearance of this ship in the sources';
......@@ -357,7 +368,7 @@ FROM
WHERE s.ship_id = k.ship_id;
update navigoviz.ship s set mindate = k.mindate, maxdate=k.maxdate, nb_sourcedoc=k.nb
update navigoviz.ship s set mindate = to_char(k.mindate, 'yyyy-mm-dd'), maxdate=to_char(k.maxdate, 'yyyy-mm-dd'), nb_sourcedoc=k.nb
from (
Select ship_id, min(p.date_fixed) as mindate, max(p.date_fixed) as maxdate, count(distinct source_doc_id) as nb
FROM navigoviz.pointcall p
......@@ -383,7 +394,7 @@ FROM
WHERE s.captain_id = k.captain_id;
update navigoviz.captain s set mindate = k.mindate, maxdate=k.maxdate, nb_sourcedoc=k.nb
update navigoviz.captain s set mindate = to_char(k.mindate, 'yyyy-mm-dd'), maxdate=to_char(k.maxdate, 'yyyy-mm-dd'), nb_sourcedoc=k.nb
from (
Select captain_id, min(p.date_fixed) as mindate, max(p.date_fixed) as maxdate, count(distinct source_doc_id) as nb
FROM navigoviz.pointcall p
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment