Add comment on tables, doc in user manual of API.

Detailled the process to build travels with net_route_marker well checked.

apidata.py

@@ -1163,8 +1163,8 @@ def getCaptainsDescription():
     Translation of birthplace in standardized en/fr is not done at that moment (15 June 2022) : there is only a birthplace_id to link with port in the ETL...
     The attribute citizenship has not its own id filled neither in navigocorpus. 
 
-    http://127.0.0.1:5004/api/ships/?format=csv&lang=fr
-    http://127.0.0.1:80/api/ships/?format=json&lang=en
+    http://data.portic.fr/api/captains/?format=csv&lang=fr
+    http://127.0.0.1:80/api/captains/?format=json&lang=en
     """
 
     

docs/usermanual.md

@@ -517,3 +517,46 @@ http://localhost/api/sources/?srid=4326&params=uhgs_id,toponym,pointcall_year,nb
 http://localhost/api/sources/?srid=4326&params=uhgs_id,toponym,pointcall_year,source_suite,nb_conges_inputdone,nb_conges_cr,point&order=toponym,pointcall_year,source_suite&format=csv
 ```
 
+# /ships?
+
+Return the ships'list that we can follow in travels 
+    Will be extracted from postgres, schema navigoviz, table ship (see sql/portic_detailviz.sql), 
+    but with a filter on the lang : only fr or en attributes
+        - lang : **fr** | en, language for flag(_standardized), homeport, and birthplace (later on)
+
+    http://127.0.0.1:80/api/ships/?format=csv&lang=fr
+    http://127.0.0.1:80/api/ships/?format=json&lang=en
+
+    Each ship attribute (name, homeport, flag, tonnageclass, class) is associated with the frequency of occurence in source documents in a JSON array format : 
+    [{"variant1 of the attribute" : frequency}, {"variant2 of the attribute" : frequency}].
+
+    For instance :
+
+    0010348N	[{"Negociant" : 1},{"Négociant de Bruges" : 1}]	[{"Bruges" : 1},{"Hambourg" : 1}]	[{"Bruges" : 1},{"Hamburg" : 1}]	[{"hambourgeois" : 1},{"Pays-Bas autrichien" : 1}]	[{"Austrian Netherlands" : 1},{"Hamburguese" : 1}]		[{"[101-200]" : 2}]
+    
+    This ship having 0010348N as identifier appears in 2 documents, named "Negociant" (once) or "Négociant de Bruges" (once). The ship's homeport is said "Bruges" in one case, "Hambourg" in the other case, and the ship's tonnage is between "101-200" tons in boths cases.
+
+    0011092N	[{"Vierge du Rosaire" : 2},{"Notre Dame du Rosaire" : 1}]	[{"Gênes" : 1}]	[{"Genoa" : 1}]	[{"savoyard" : 2},{"génois" : 1}]	[{"Savoyard" : 2},{"Genoese" : 1}]	[{"bateau" : 2},{"tartane" : 1}]	
+
+    This ship having 0011092N as identifier appears in 3 documents, named "Vierge du Rosaire" (twice) or "Notre Dame du Rosaire" (once). The ship's homeport is said "Gênes" in one case, "savoyard" in the other case, and the ship's tonnage is unknown but the ship's class is said to be "bateau" in two documents, "tartane" in the third document.
+
+# /captains?
+
+Return the captains'list that we can follow in travels 
+    Will be extracted from postgres, schema navigoviz, table captain (see sql/portic_detailviz.sql), 
+    but with a filter on the lang : only fr or en attributes
+        - lang : **fr** | en, language for flag(_standardized), homeport, and birthplace (later on)
+
+    http://127.0.0.1:80/api/ships/?format=csv&lang=fr
+    http://127.0.0.1:80/api/ships/?format=json&lang=en
+
+    Each captain attribute (name, citizenship, birthplace) is associated with the frequency of occurence in source documents in a JSON array format : 
+    [{"variant1 of the attribute" : frequency}, {"variant2 of the attribute" : frequency}].
+
+    For instance :
+
+    00014569	[{"Accamo, Jean Baptiste" : 18},{"Accamo,Jean Baptiste" : 1},{"Accame, Jean Baptiste" : 1},{"Accamé, Jean Baptiste" : 1},{"Accamo, Jean BAptiste" : 1}]		[{"Génois" : 18},{"Genois" : 3}]
+
+    This captain idendified by the number 00014569 has been found named "Accamo, Jean Baptiste" 18 times, but only once named "Accame, Jean Baptiste". In 18 documents, he was a Génois, in 3 he was a Genois. The citizenship attribute has not been yet standardized. 
+
+    
\ No newline at end of file

sql/portic_detailviz.sql

@@ -15,9 +15,10 @@ p.citizenship , p.citizenship_uncertainity
 --captain_citizenship, captain_birthplace, captain_birthplace_id, captain_origin, captain_origin_id 
 from  navigoviz.built_travels p
 where source_entry != 'both-to'
-order by captain_id
+order by captain_id;
 -- 21994
-drop table navigoviz.captain;
+
+-- drop table navigoviz.captain;
 
 CREATE TABLE navigoviz.captain (
 		captain_id text,
@@ -49,7 +50,7 @@ GROUP BY captain_local_id
 WHERE c.captain_id = k.captain_local_id;
 -- 12329 
 
-select * from navigoviz.captain where occurences_names is null;
+-- select * from navigoviz.captain where occurences_names is null;
 
 update navigoviz.captain c set occurences_birthplaces = k.occurence_birthplace
 FROM
@@ -228,10 +229,10 @@ FROM
 (SELECT t.ship_id, array_to_json(array_agg(json_build_object(t.ship_class ,t.occurences))) as class
 	FROM(
 		Select ship_id,
-		class as ship_class, count (distinct source_doc_id) as occurences
+		trim(lower(class)) as ship_class, count (distinct source_doc_id) as occurences
 		FROM navigoviz.pointcall p
 	WHERE p.ship_id in (select distinct ship_id from built_travels)
-		GROUP BY ship_id, class
+		GROUP BY ship_id, trim(lower(class))
 		ORDER BY ship_id, occurences DESC
 		) t
 	 WHERE occurences > 0 and ship_class is not null
@@ -264,12 +265,117 @@ select * from navigoviz.ship where  json_array_length (occurences_tonnageclass)
 select * from navigoviz.ship where  json_array_length (occurences_tonnageclass) > 1 --622
 select (occurences_tonnageclass)::json->0->>'f1' from navigoviz.ship where occurences_tonnageclass is not null
 
-select travel_id, source_doc_id, outdate_fixed, indate_fixed, departure_uhgs_id, departure_fr, destination_uhgs_id, destination_fr,
+select travel_id as step_id, source_doc_id, outdate_fixed, indate_fixed, departure_uhgs_id, departure_fr, destination_uhgs_id, destination_fr,
 ship_id, ship_name, tonnage_class, ship_flag_standardized_fr, homeport_toponyme_fr, 
-captain_id , captain_name, birthplace, citizenship, travel_rank, nbproduits
+captain_id , captain_name, birthplace, citizenship, travel_rank
 from navigoviz.built_travels
-where ship_id = '0013102N' and source_entry!='both-to'
+where ship_id = '0022221N' and source_entry!='both-to'
 order by travel_rank;
 -- and captain_id=''
 
+
 -- Renvoyer les variantes de noms de navires ou de capitaines
+GRANT USAGE ON SCHEMA navigoviz TO api_user;
+GRANT USAGE ON SCHEMA navigocheck to api_user;
+GRANT USAGE ON SCHEMA navigo to api_user;
+GRANT USAGE ON SCHEMA public to api_user;
+GRANT USAGE ON SCHEMA ports to api_user;
+GRANT SELECT ON ALL TABLES IN SCHEMA navigoviz, navigo, navigocheck, ports, public TO api_user;
+grant CONNECT on database portic_v7 to api_user;
+grant SELECT on all sequences in schema navigoviz, navigo, navigocheck, ports, public to api_user;
+
+-------------------------------------------------------------------
+-------------------------------------------------------------------
+
+-- Travail sur les trajectoires
+
+-- Aimable Ursule - ship 0007498N 
+-- Fidèle Marianne - ship 0002931N
+-- Marie Louise - ship 0004155N
+-- Turgot, ship id 0021517N
+-- Femmes et Enfants, 0003951N
+-- Notre Dame des Carmes 0004171N
+
+
+select travel_id as step_id, source_text, outdate_fixed, indate_fixed, departure_uhgs_id, departure_fr, destination_uhgs_id, destination_fr,
+ship_id, ship_name, tonnage_class, ship_flag_standardized_fr, homeport_toponyme_fr, 
+captain_id , captain_name, birthplace, citizenship, travel_rank
+from navigoviz.built_travels
+where ship_id = '0004171N' and source_entry!='both-to'
+order by travel_rank;
+
+comment on table navigoviz.ship is 'List of ships''descriptions with variantes for the name, flag, homeport, tonnage_class and class of ship';
+comment on table navigoviz.captain is 'List of captains''descriptions with variantes for the name, citizenship, birthplace of captain';
+
+comment on column navigoviz.ship.ship_id is 'unique identifier of ship set by Silvia who identifies the same ships through out various source documents';
+comment on column navigoviz.ship.occurences_names is 'JSON array of various names of the same ship through sources, with the frequecy of appearance in documents';
+comment on column navigoviz.ship.occurences_homeports_fr is 'JSON array of various homeports (standardized name in FR lang) of the same ship through sources, with the frequecy of appearance in documents';
+comment on column navigoviz.ship.occurences_homeports_en is 'JSON array of various homeports (standardized name in EN lang) of the same ship through sources, with the frequecy of appearance in documents';
+comment on column navigoviz.ship.occurences_flags_fr is 'JSON array of various flag (standardized name in FR lang) of the same ship through sources, with the frequecy of appearance in documents';
+comment on column navigoviz.ship.occurences_flags_en is 'JSON array of various flag (standardized name in EN lang) of the same ship through sources, with the frequecy of appearance in documents';
+comment on column navigoviz.ship.occurences_class is 'JSON array of various class types of the same ship through sources, with the frequecy of appearance in documents';
+comment on column navigoviz.ship.occurences_tonnageclass is 'JSON array of various categories of tonnage of the same ship through sources, with the frequecy of appearance in documents';
+
+comment on column navigoviz.captain.captain_id is 'unique identifier of captain set by Silvia who identifies the same captain through out various source documents';
+comment on column navigoviz.captain.occurences_names is 'JSON array of various names of the same captain through sources, with the frequecy of appearance in documents';
+comment on column navigoviz.captain.occurences_birthplaces is 'JSON array of various birthplace of the same captain through sources, with the frequecy of appearance in documents';
+comment on column navigoviz.captain.occurences_citizenships is 'JSON array of various citizenship of the same captain through sources, with the frequecy of appearance in documents';
+
+
+	
+-----------------------------------------------------------------------------------------
+-- Reprise du travail sur maritime_paths_21fev2022
+---------------------------------------------------------------------------------------------
+
+-- 1. Enrichir navigoviz.pointcall avec nouveaux attributs utilisés par l'algo pour vérifier les A/Z
+-- date_precise : (case when position('=' in coalesce(p.pointcall_out_date , p.pointcall_in_date)) > 0 then true else false end) as date_precise,
+-- list_sup_uhgs_id : 
+-- list_inf_uhgs_id : 
+-- has_input_done : vrai si il y a des sources pour ce port dans le G5 (faux à l'étranger toujours, et vrai pour Marseille)
+/* rajouter ces attributs dans les commentaires des tables pour l'API pointcall */
+-- fait sur le serveur le 15 juin 2022
+
+-- 2. L'algo va créer public.pointcall_verifs
+-- 3. On demande ensuite la création de navigoviz.pointcall_checked (public.pointcall_heloise) sont la colonne net_route_marker est corrigée
+    -- et qui surtout qualifie le niveau de certitude qu'on a qu'un pointcall ait été fréquenté ou pas
+-- 4. On corrige la fabrique de built_travels dans l'ETL (heloise_historiens) par autojointure sur navigoviz.pointcall_checked
+-- 5. On fait fabriquer à l'algo la table invalidated_travels dans l'ETL (heloise_algo) 
+	 -- en calculant l'autojointure sur navigoviz.pointcall_checked en retenant seulement les troncons avec -5 (INFIRMED) à un bout ou l'autre
+-- 6. On calcule les couleurs et le style de troncons de built_travels (pt de vue historien) et de invalidated_travels (pt de vue algo)
+	-- on calcule aussi la trajectoire parcourue dans pointpath
+-- 7. On fait l'union de ces 2 tables built_travels + invalidated_travels
+    -- et on donne un identifiant à chaque tronçon pour fabriquer un geosjon plus facilement : row_number() over () as tid
+
+
+alter table navigoviz.pointcall add column date_precise boolean;
+update navigoviz.pointcall p set date_precise =  (case when position('=' in coalesce(p.pointcall_out_date , p.pointcall_in_date)) > 0 then true else false end) ;
+COMMENT ON COLUMN navigoviz.pointcall.date_precise IS 'For algo : true indicates that the date of the pointcall is precisely known ';
+
+alter table navigoviz.pointcall add column has_input_done boolean;
+update navigoviz.pointcall p set has_input_done =  (case when p.state_1789_fr = 'France' then
+                (case when p.source_suite != 'G5' then
+                    true
+                else
+                    (case when extract(year from coalesce(p.outdate_fixed, p.indate_fixed)) = 1787 then
+                        nb_conges_1787_inputdone is not null and p.nb_conges_1787_inputdone > 0
+                    else p.nb_conges_1789_inputdone is not null and p.nb_conges_1789_inputdone > 0
+                    end)
+                end)
+                else false
+                end);
+COMMENT ON COLUMN navigoviz.pointcall.has_input_done IS 'For algo : true indicates that there is sources (G5 registries) for this pointcall';
+
+alter table navigoviz.pointcall add column list_sup_uhgs_id _varchar;
+alter table navigoviz.pointcall add column list_inf_uhgs_id _varchar;
+
+COMMENT ON COLUMN navigoviz.pointcall.list_sup_uhgs_id IS 'For algo : list of entities whom pointcall could geographically belong to';
+COMMENT ON COLUMN navigoviz.pointcall.list_inf_uhgs_id IS 'For algo : list of entities that could geographically belong to this pointcall';
+
+update navigoviz.pointcall p set list_sup_uhgs_id = k.list_sup_uhgs_id
+from (select ughs_id, array_agg(ughs_id_sup) as list_sup_uhgs_id
+            from ports.generiques_inclusions_geo_csv gigc
+            group by ughs_id) as k where k.ughs_id = p.pointcall_uhgs_id;
+update navigoviz.pointcall p set list_inf_uhgs_id = k.list_inf_uhgs_id
+from (select ughs_id_sup, array_agg(ughs_id) as list_inf_uhgs_id
+            from ports.generiques_inclusions_geo_csv gigc
+            group by ughs_id_sup) as k where k.ughs_id_sup = p.pointcall_uhgs_id;            
\ No newline at end of file