cambio endpoint del trasporto (cls 2007) e nuovo metadata file (issue #…

…15 e #21)

terra-update-batch/execute/output.py

@@ -85,21 +85,22 @@ def exectuteOutput(logger):
  except Exception as e:
  logger.error(f'Error creating CPA intra and CPA product code files: {str(e)}')
 
- #[PYTHON-SERVER] Creazione file TR extra-UE e TR product code
+ #[PYTHON-SERVER] Creazione file TR extra-UE, TR product code e TR trimestrali
  try:
  output_interval["graphExtra"] = cOut.createOutputGraphExtraUE(
  input_path = params.DIRECTORIES["TRANSPORT_MONTHLY_FILE"],
  output_tr_extra_ue_file = params.FILES["TR_EXTRA_UE_CSV"],
  output_tr_prod_code_file = params.FILES["TR_PRODUCT_CODE_CSV"],
+ output_tr_extra_ue_trim = params.FILES["TR_EXTRA_UE_TRIMESTRALI_CSV"],
  logger = logger
  )
  logger.info(f'Created TR extra and TR product code files')
  except Exception as e:
- logger.error(f'Error creating TR extra and TR product code files: {str(e)}')
+ logger.error(f'Error creating TR extra, TR product code files and TR trimestrali: {str(e)}')
 
  #[PYTHON-SERVER] Creazione file CPA trim
  try:
- output_interval["graphIntraTrim"] = cOut.createOutputGraphicTrimestre(
+ output_interval["graphIntraTrim"] = cOut.createOutputGraphTrimestre(
  db = params.FILES["SQLLITE_DB"],
  output_cpa_trim = params.FILES["CPA_TRIM_CSV"],
  logger = logger
@@ -108,17 +109,6 @@ def exectuteOutput(logger):
  except Exception as e:
  logger.error(f'Error creating quarterly CPA file: {str(e)}')
 
- #[PYTHON-SERVER] Creazione file TR extra-UE trim
- try:
- output_interval["graphExtraTrim"] = cOut.createOutputGraphExtraUE_Trim(
- input_path = params.DIRECTORIES["TRANSPORT_MONTHLY_FILE"],
- output_tr_extra_ue_trim = params.FILES["TR_EXTRA_UE_TRIMESTRALI_CSV"],
- logger = logger
- )
- logger.info(f'Created quarterly TR extra file')
- except Exception as e:
- logger.error(f'Error creating quarterly TR extra file: {str(e)}')
-
  #[R-SERVER] Creazione file Comext IMP/EXP e CPA2 product code
  try:
  output_interval["variazioniCPA"] = cOut.createOutputVariazioniQuoteCPA(

terra-update-batch/modules/cosmoDownload.py

@@ -79,7 +79,7 @@ def data_download(self, frequency: str, file_type: str, url_download: str, out_p
 
  Args:
  frequency: The frequency of the files to be downloaded (monthly or annual).
- file_type: The type of files to be downloaded ('full' or 'tr').
+ file_type: The type of files to be downloaded ('full' or 'nst07_extra').
  url_download: The url of the files to be downloaded.
  out_path: The path where the extracted files will be saved.
  zip_path: The path where the zipped files will be saved.

terra-update-batch/modules/cosmoOutput.py

@@ -1087,7 +1087,7 @@ def createOutputGraphCPAIntraUE(db, cpa_intra, cpa3_prod_code, logger):
  #return "CPA Graphic INTRA UE OK; files created: " + cpa_intra
 
 
-def createOutputGraphicTrimestre(db, output_cpa_trim, logger):
+def createOutputGraphTrimestre(db, output_cpa_trim, logger):
  logger.info("createOutputGraphicTrimestre START")
  # import export variazioni quote CPA
  logger.info("import export variazioni quote CPA INTRA TRim")
@@ -1109,7 +1109,7 @@ def createOutputGraphicTrimestre(db, output_cpa_trim, logger):
  #return "CPA Graphic INTRA TRIMESTRE UE OK; files created: " + output_cpa_trim
 
 
-def createOutputGraphExtraUE(input_path, output_tr_extra_ue_file, output_tr_prod_code_file, logger):
+def createOutputGraphExtraUE(input_path, output_tr_extra_ue_file, output_tr_prod_code_file, output_tr_extra_ue_trim, logger):
  logger.info("createOutputGraphExtraUE START")
  logger.info("Reading from " + input_path)
 
@@ -1126,12 +1126,12 @@ def createOutputGraphExtraUE(input_path, output_tr_extra_ue_file, output_tr_prod
  listDataframes.append(appo)
 
  df = pd.concat(listDataframes, axis=0)
- # df=df[df["PRODUCT_NSTR"]!="TOT"]
+ # df=df[df["PRODUCT_NST07"]!="TOT"]
  df = df[df["DECLARANT_ISO"] != "EU"]
  df = df[df["PARTNER_ISO"] != "EU"]
  df = df[
  [
- "PRODUCT_NSTR",
+ "PRODUCT_NST07",
  "DECLARANT_ISO",
  "PARTNER_ISO",
  "PERIOD",
@@ -1141,55 +1141,28 @@ def createOutputGraphExtraUE(input_path, output_tr_extra_ue_file, output_tr_prod
  "QUANTITY_IN_KG",
  ]
  ]
-
+ df.rename(columns={'PRODUCT_NST07': 'PRODUCT_NSTR'}, inplace=True)
+ df_tot = df.groupby([
+ "DECLARANT_ISO",
+ "PARTNER_ISO",
+ "PERIOD",
+ "TRANSPORT_MODE",
+ "FLOW"
+ ]).sum().reset_index()[["DECLARANT_ISO","PARTNER_ISO","PERIOD","TRANSPORT_MODE","FLOW","VALUE_IN_EUROS","QUANTITY_IN_KG"]]
+ df_tot["PRODUCT_NSTR"] = "TOT"
+ df = pd.concat([df, df_tot], ignore_index=True)
  df.to_csv(output_tr_extra_ue_file, sep=",", index=False)
 
+ # PRODUCT CODE
  pd.DataFrame({"PRODUCT": df["PRODUCT_NSTR"].astype(str).unique()}).to_csv(
  output_tr_prod_code_file, sep=",", index=False
  )
 
  logger.info("tr_extra_ue file: " + output_tr_extra_ue_file)
  logger.info("createOutputGraph END ")
-
- time_range = list(df["PERIOD"].agg(['min', 'max']))
- return time_range
- #return "CPA Graphic EXTRE UE OK; files created: " + output_tr_extra_ue_file
-
-
-def createOutputGraphExtraUE_Trim(input_path, output_tr_extra_ue_trim, logger):
+
+ # TRIMESTRALI
  logger.info("createOutputGraphExtraUE TRIM START")
- logger.info("Reading from " + input_path)
-
- listDataframes = []
-
- for f in os.listdir(input_path):
- if f.endswith(params.DATA_EXTENTION):
- appo = pd.read_csv(
- input_path + os.sep + f,
- sep=params.SEP,
- low_memory=False,
- keep_default_na=False,
- na_values=[""],
- )
- listDataframes.append(appo)
-
- df = pd.concat(listDataframes, axis=0)
- # df=df[df["PRODUCT_NSTR"]!="TOT"]
- df = df[df["DECLARANT_ISO"] != "EU"]
- df = df[df["PARTNER_ISO"] != "EU"]
- df = df[
- [
- "PRODUCT_NSTR",
- "DECLARANT_ISO",
- "PARTNER_ISO",
- "PERIOD",
- "TRANSPORT_MODE",
- "FLOW",
- "VALUE_IN_EUROS",
- "QUANTITY_IN_KG",
- ]
- ]
-
  df["TRIMESTRE"] = df["PERIOD"]
  df["TRIMESTRE"] = df["TRIMESTRE"].apply(
  lambda x: str(x)[0:4] + "T1"
@@ -1222,10 +1195,9 @@ def createOutputGraphExtraUE_Trim(input_path, output_tr_extra_ue_trim, logger):
  logger.info("tr_extra_ue TRIMESTRALI file: " + output_tr_extra_ue_trim)
  logger.info("createOutputGraph TRIM END ")
 
- time_range = list(df_trim["TRIMESTRE"].agg(['min', 'max']))
+ time_range = list(df["PERIOD"].agg(['min', 'max']))
  return time_range
- #return "CPA Graphic EXTRE UE OK; files created: " + output_tr_extra_ue_trim
-
+ #return "CPA Graphic EXTRE UE OK; files created: " + output_tr_extra_ue_file
 
 def createClsNOTEmptyProductsLang(
  digit, langs, clsfiles, filename, filterValue, fileExistingProducts, logger

terra-update-batch/resources/params.py

@@ -9,7 +9,7 @@
 RUN_UTILS : bool = True
 
 WORKING_FOLDER : str = os.environ['WORKING_FOLDER']
-#WORKING_FOLDER : str = "C:" + os.sep + "Users" + os.sep + "UTENTE" + os.sep + "terra_output"
+#WORKING_FOLDER  : str = "C:" + os.sep + "Users" + os.sep + "UTENTE" + os.sep + "terra_output"
 
 KEY_VAULT_NAME : str = "statlab-key-vault"
 SECRETNAME_ACCOUNTKEY : str = "cosmostoragekey"
@@ -38,11 +38,10 @@
 PAGE_GRAPH_INTRA_UE_TIME_INTERVAL_M: int = 60
 PAGE_BASKET_TIME_INTERVAL_M : int = 60
 
-
 PREFIX_PRODUCT : str = "full"
-PREFIX_TRANSPORT: str = "tr"
+PREFIX_TRANSPORT: str = "nst07_extra"
 PREFIX_MAP : dict[str] = {
- "tr": "transport",
+ "nst07_extra": "transport",
  "full": "product"
 }
 
@@ -126,16 +125,12 @@
  "ANNUAL_UNEMPLOYEMENT" : "https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/UNE_RT_A/?format=SDMX-CSV&i",
 
  "COMEXT_PRODUCTS" : "https://ec.europa.eu/eurostat/api/dissemination/files/?sort=1&downfile=comext%2FCOMEXT_DATA%2FPRODUCTS%2F",
-
- # NEW ENDPOINT: https://ec.europa.eu/eurostat/api/dissemination/files/?sort=1&dir=comext%2FCOMEXT_DATA%2FTRANSPORT_NSTR
- # ACTUAL USED ENDPOINT: "https://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&file=comext%2FCOMEXT_DATA%2FTRANSPORT_NSTR%2F"
- # "https://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing?sort=1&file=comext%2FCOMEXT_HISTORICAL_DATA%2FTRANSPORT_BY_NSTR%2F"
- "COMEXT_TR" : "https://ec.europa.eu/eurostat/api/dissemination/files?sort=1&file=comext%2FCOMEXT_HISTORICAL_DATA%2FTRANSPORT_BY_NSTR%2F",
+ "COMEXT_TR" : "https://ec.europa.eu/eurostat/api/dissemination/files/?sort=1&file=comext%2FCOMEXT_DATA%2FTRANSPORT_NST07%2F",
 
  "CLS_PRODUCTS" : "https://ec.europa.eu/eurostat/api/dissemination/files/?sort=1&file=comext%2FCOMEXT_METADATA%2FCLASSIFICATIONS_AND_RELATIONS%2FCLASSIFICATIONS%2FENGLISH%2FCN.txt",
 
- "CLS_NSTR" : "https://ec.europa.eu/eurostat/api/dissemination/files/?sort=1&file=comext%2FCOMEXT_METADATA%2FCLASSIFICATIONS_AND_RELATIONS%2FCLASSIFICATIONS%2FENGLISH%2FNSTR.txt",
- "CLS_NSTR_ITA" : "https://raw.githubusercontent.com/istat-methodology/terra/main/cls/Prodotti_NSTR_ita.csv",
+ "CLS_NSTR" : "https://ec.europa.eu/eurostat/api/dissemination/files/?sort=1&file=comext%2FCOMEXT_METADATA%2FCLASSIFICATIONS_AND_RELATIONS%2FCLASSIFICATIONS%2FENGLISH%2FNST2007.txt",
+ "CLS_NSTR_ITA" : "https://raw.githubusercontent.com/istat-methodology/terra/main/cls/Prodotti_NSTR_2007_ita.csv",
 
  "CLS_CPA" : "https://ec.europa.eu/eurostat/api/dissemination/files/?sort=1&file=comext%2FCOMEXT_METADATA%2FCLASSIFICATIONS_AND_RELATIONS%2FCLASSIFICATIONS%2FENGLISH%2FCPA21.txt",
  "CLS_CPA_3D_ITA" : "https://raw.githubusercontent.com/istat-methodology/terra/main/cls/CPA_2_1_3digits_ita.csv",