Commit f89db138 authored by Christine Plumejeaud's avatar Christine Plumejeaud
Browse files

Data of April 2020

ETL for building an API publishing navigo data : pointcalls and travels.
parent 7a6f6e4e
This diff is collapsed.
......@@ -21,7 +21,7 @@ from stat import *
import random
from xlrd import open_workbook, cellname, XL_CELL_TEXT, XL_CELL_DATE, XL_CELL_BLANK, XL_CELL_EMPTY, XL_CELL_NUMBER, \
xldate_as_tuple
#pyopenxls à voir
## pour avoir le path d'un package
## print (psycopg2.__file__)
......@@ -107,33 +107,41 @@ class LoadFilemaker(object):
'''
self.loadfile(config, 'geo_general')
self.loadfile(config, 'pointcall')
self.loadfile(config, 'acting_parties')
self.loadfile(config, 'actions')
self.loadfile(config, 'taxes')
self.loadfile(config, 'cargo')
self.loadfile(config, 'component_description')
self.create_uncertain_check_tables(config, 'geo_general')
self.populate_uncertain_check_tables(config, 'geo_general')
'''
self.loadfile(config, 'pointcall')
self.create_uncertain_check_tables(config, 'pointcall')
self.create_uncertain_check_tables(config, 'acting_parties')
self.create_uncertain_check_tables(config, 'actions')
self.create_uncertain_check_tables(config, 'taxes')
self.create_uncertain_check_tables(config, 'cargo')
self.create_uncertain_check_tables(config, 'component_description')
self.populate_uncertain_check_tables(config, 'pointcall')
self.populate_uncertain_check_tables(config, 'acting_parties')
self.populate_uncertain_check_tables(config, 'actions')
self.populate_uncertain_check_tables(config, 'taxes')
#self.loadfile(config, 'acting_parties')
#self.loadfile(config, 'actions')
#self.loadfile(config, 'component_description')
self.loadfile(config, 'cargo')
self.create_uncertain_check_tables(config, 'cargo')
self.populate_uncertain_check_tables(config, 'cargo')
self.populate_uncertain_check_tables(config, 'component_description')
'''
self.loadfile(config, 'taxes')
self.create_uncertain_check_tables(config, 'taxes')
self.populate_uncertain_check_tables(config, 'taxes')
#self.create_uncertain_check_tables(config, 'component_description')
#self.create_uncertain_check_tables(config, 'acting_parties')
#self.create_uncertain_check_tables(config, 'actions')
#self.populate_uncertain_check_tables(config, 'acting_parties')
#self.populate_uncertain_check_tables(config, 'actions')
#self.populate_uncertain_check_tables(config, 'component_description')
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
print(e.message)
print(e)
print(repr(traceback.format_exception(exc_type, exc_value, exc_traceback)))
self.logger.error(e)
self.logger.error(e.message)
......@@ -145,7 +153,7 @@ class LoadFilemaker(object):
sql_createtable = """SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA='navigo' and TABLE_NAME = '%s' order by ordinal_position""" % (
relation)
self.logger.info(sql_createtable)
columns = self.select_sql(sql_createtable);
columns = self.select_sql(sql_createtable)
sql_createtable = """ CREATE TABLE navigocheck.uncertainity_""" + relation + """ ( """
for c in columns:
......@@ -154,7 +162,7 @@ class LoadFilemaker(object):
sql_createtable += ");"
self.logger.info(sql_createtable)
sql_droptable = "drop table IF EXISTS navigocheck.uncertainity_" + relation + " cascade;"
self.execute_sql(sql_droptable);
self.execute_sql(sql_droptable)
self.execute_sql(sql_createtable)
sql_createtable = """ CREATE TABLE navigocheck.check_""" + relation + """ ( """
......@@ -166,8 +174,9 @@ class LoadFilemaker(object):
sql_createtable = sql_createtable[:-1]
sql_createtable += ");"
self.logger.info(sql_createtable)
sql_droptable = "drop table IF EXISTS navigo.check_" + relation + " cascade;"
sql_droptable = "drop table IF EXISTS navigocheck.check_" + relation + " cascade;"
self.execute_sql(sql_droptable);
print(sql_droptable)
self.execute_sql(sql_createtable)
def populate_uncertain_check_tables(self, config, relation):
......@@ -186,7 +195,7 @@ class LoadFilemaker(object):
sql_insert_head = sql_insert_head[:-1]
sql_insert_head += " from navigo." + relation+");"
self.logger.info("\n"+sql_insert_head+"\n")
#self.execute_sql(sql_insert_head)
self.execute_sql(sql_insert_head)
sql_insert_head = "insert into navigocheck.check_" + relation + " ( select "
......@@ -198,7 +207,7 @@ class LoadFilemaker(object):
sql_insert_head = sql_insert_head[:-1]
sql_insert_head += " from navigo." + relation+");"
self.logger.info("\n"+sql_insert_head+"\n")
#self.execute_sql(sql_insert_head)
self.execute_sql(sql_insert_head)
def loadfile(self, config, relation):
print('Processing ' + relation)
......@@ -267,7 +276,7 @@ class LoadFilemaker(object):
cur.execute(sql_query)
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
print(e.message)
print(e)
print(repr(traceback.format_exception(exc_type, exc_value, exc_traceback)))
self.logger.error(sql_query)
......@@ -281,7 +290,7 @@ class LoadFilemaker(object):
return cur.fetchall()
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
print(e.message)
print(e)
print(repr(traceback.format_exception(exc_type, exc_value, exc_traceback)))
self.logger.error(sql_query)
......@@ -292,7 +301,7 @@ class LoadFilemaker(object):
if __name__ == '__main__':
# Passer en parametre le nom du fichier de configuration
# configfile = sys.argv[1]
configfile = 'config_loadfilemaker.txt'
configfile = 'config_loadfilemaker_plume.txt'
config = configparser.RawConfigParser()
config.read(configfile)
......
##
## Auteur : Christine Plumejeaud, 13 février 2019
## FICHIER DE CONFIGURATION pour l'intégration des données de navigo dans leur format original (export CSV de filemaker)
## A utiliser avec : LoadFilemaker.py, BuildNavigoviz.py, Controls.py
##
#################################################################################################################
##
## Coordonnées de la base postgres dans laquelle on importe les données
##
#################################################################################################################
[base]
host=localhost
port=5433
dbname=portic_v3
user=navigo
password=navigocorpus2018
#user=postgres
#password=postgres
#################################################################################################################
##
## SSH config
##
#################################################################################################################
[ssh]
server=134.158.33.179
port=22
postgres_server=localhost
postgres_port=5432
# Note : postgres_server = 'localhost' = base.host
## User that is logging (psauriau should be ok)
user=navigo
## Path to your private key (RSA) for connecting on SSH server - SHOULD NEVER be given - generated from PUTTYGen
#ppk=C:\\Users\\cplume01\\Travail\\CNRS\\Travail_LIENSs\\Projets\\Pierre-Guy_Sauriau\\Dev\\privateKey_psauriau.ppk
ppk=F:\\Dev\\Marinpertuis\\adminsite\\marinpertuis\\dist\\Data\\SSH\\privateKey_psauriau.ppk
## Path to the open SSH private key used by paramiko for connecting SSH
#opensshppk=C:\\Users\\cplume01\\Travail\\CNRS\\Travail_LIENSs\\Projets\\Pierre-Guy_Sauriau\\Dev\\id_dsa_com_psauriau.pub
opensshppk=F:\\Dev\\Marinpertuis\\adminsite\\marinpertuis\\dist\\Data\\SSH\\\\id_dsa_com_psauriau.pub
## Pass phrase to open your openSSH private key (RSA) for connecting on SSH server - SHOULD NEVER be given
passwd=escargot2mer2016
## Path to Putty (MUST be downloaded before: http://www.chiark.greenend.org.uk/~sgtatham/putty/download.html)
putty=C:\\Program Files (x86)\\PuTTY\\putty.exe
## Path to WinSCP (MUST be downloaded and installed before: https://winscp.net/download/winscp577setup.exe)
winscp=C:\\Program Files (x86)\\WinSCP\\winscp.com
#################################################################################################################
##
## Fichiers de log
## https://docs.python.org/2/library/logging.html
## levels
## ERROR 40
## WARNING 30
## INFO 20
## DEBUG 10
##
#################################################################################################################
[log]
# Fichier de log pour garder une trace des opérations réalisées (il écrase le précédent si il porte le même nom)
#file=C:\Travail\Dev\portic\ETL\log_29dec2019c.txt
#file=/home/plumegeo/navigo/ETL/log_14fev2019.txt
file=/home/plumegeo/navigo/ETL/data_14mai2020/log_14mai2020.txt
level=10
#################################################################################################################
##
## HTTP config
##
#################################################################################################################
[proxy]
## At ULR, you have to use proxy (True), else not (False)
need=False
## Adresse du proxy : IP:port
http_proxy=10.1.30.18:3128
#################################################################################################################
##
## Navigo sources
##
#################################################################################################################
[geo_general]
file_name=C:\\Travail\\ULR_owncloud\\ANR_PORTIC\\Data\\ports\\AppartenanceEtat_20fev2020\\pointcall_G5_Marseille_1787 USGH lat long.xlsx
sheet_name=Sheet1
filtre=True
[pointcall]
file_name=/home/plumegeo/navigo/ETL/data_14mai2020/pointcall_all_fields_fr_1787_26avril2020.xlsx
sheet_name=pointcall
filtre=(source like '%G5%' or source like 'ADBdR%' or source like 'ADVar%' or source like '%G-5%')
[acting_parties]
#file_name=C:\\Travail\\ULR_owncloud\\ANR_PORTIC\\IDEX\\BDD_navigo\\Version_24dec2019\\navigo_20191224.xlsx
#sheet_name=acting_parties
#filtre=True
[actions]
#file_name=C:\\Travail\\ULR_owncloud\\ANR_PORTIC\\IDEX\\BDD_navigo\\Version_24dec2019\\navigo_20191224.xlsx
#sheet_name=actions
#filtre=True
[taxes]
file_name=/home/plumegeo/navigo/ETL/data_14mai2020/Taxes_all_fields.xlsx
sheet_name=Sheet1
filtre=(source like '%G5%' or source like 'ADBdR%' or source like 'ADVar%' or source like '%G-5%')
[cargo]
file_name=/home/plumegeo/navigo/ETL/data_14mai2020/pointcall_cargo_all_fields.xlsx
sheet_name=Sheet1
filtre=(source like '%G5%' or source like 'ADBdR%' or source like 'ADVar%' or source like '%G-5%')
[component_description]
#file_name=D:\\CNRS\\Travail_LIENSs\\Projets\\ANR_PORTIC\\IDEX\\BDD_navigo\\component_description 22 octobre 2018.xlsx
file_name=/home/plumegeo/navigo/ETL/data_20191023/component_description 22 octobre 2018.xlsx
sheet_name=Sheet1
#################################################################################################################
##
## Check outputs
##
#################################################################################################################
[outputs]
#file_name=C:\\Travail\\ULR_owncloud\\ANR_PORTIC\\IDEX\\BDD_navigo\\Controles\\analysis_24dec2019.xlsx
file_name=/home/plumegeo/navigo/ETL/data_14mai2020/debug.xlsx
#################################################################################################################
##
## Navigoviz
##
#################################################################################################################
[navigoviz]
source=delete from navigoviz.source where data_block_local_id = '00149228' and source = 'ANF, G5-62/rafraichi'
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment