fixes to migration scripts for data warehouse

bin/migrate/migrate_legacy_consultations_2019_02.py → bin/migrate/import_consultations_2019_02.py

@@ -119,6 +119,9 @@ def dt(legacy_date):
     return d.strftime('%Y-%m-%d')
 
 def main():
+    assert sys.stdin.read(3) == codecs.BOM_UTF8
+    sys.stdout.write(codecs.BOM_UTF8)
+
     try:
         in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
 
@@ -153,7 +156,6 @@ def main():
             except KeyError as err:
                 sys.stderr.write(line['ID'] + ': ' + str(err) + '\n')
 
-        sys.stdout.write(codecs.BOM_UTF8)
         out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')
         out_csv.writeheader()
         for o in sorted(orgs):

bin/migrate/migrate_ati_2019_11.py

@@ -21,7 +21,7 @@
 
         out_csv.writerow(line)
 
-except KeyError:
+except (KeyError, ValueError):
     if 'warehouse' in sys.argv:
         sys.exit(85)
     else:

bin/migrate/migrate_ati_2020_12.py

@@ -6,8 +6,13 @@
 import json
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
+
+if 'warehouse' in sys.argv and 'comments_en' in in_csv.fieldnames:
+    sys.exit(85)
+
 if 'report' in sys.argv:
     out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=['old_disposition', 'new_disposition'])
     seen = set()
@@ -77,4 +82,10 @@
 
     line['comments_en'] = ''
     line['comments_fr'] = ''
-    out_csv.writerow(line)
+    try:
+        out_csv.writerow(line)
+    except (KeyError, ValueError):
+        if 'warehouse' in sys.argv:
+            sys.exit(85)
+        else:
+            raise

bin/migrate/migrate_ati_nil_2019_11.py

@@ -8,6 +8,7 @@
 FIELDNAMES = 'year,month,record_created,record_modified,user_modified,owner_org,owner_org_title'.split(',')
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')

bin/migrate/migrate_consultations_2018_07.py

@@ -8,10 +8,10 @@
 FIELDNAMES = 'registration_number,publishable,partner_departments,subjects,title_en,title_fr,description_en,description_fr,target_participants_and_audience,start_date,end_date,status,profile_page_en,profile_page_fr,report_available_online,report_link_en,report_link_fr,contact_email,policy_program_lead_email,remarks_en,remarks_fr,high_profile,rationale,record_created,record_modified,user_modified,owner_org,owner_org_title'.split(',')
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')
-sys.stdout.write(codecs.BOM_UTF8)
 out_csv.writeheader()
 
 RATIONALE = {

bin/migrate/migrate_consultations_2019_04.py

@@ -2,23 +2,26 @@
 """
 This is an "update" script not a migrate script because it only
 outputs records to be updated in-place, not a complete migrated
-copy of the data
+copy of the data, unless "warehouse" parameter is given
 """
 
 import unicodecsv
 import sys
 import codecs
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
+if not in_csv.fieldnames and 'warehouse' in sys.argv:
+    sys.exit(85)
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8')
 out_csv.writeheader()
 
 try:
     for line in in_csv:
         val = line['subjects'].split(',')
-        if 'AP' not in val:
+        if 'AP' not in val and 'warehouse' not in sys.argv:
             continue
         line['subjects'] = u','.join(
             'IP' if v == 'AP' else v for v in val)

bin/migrate/migrate_contracts_2016_06.py

@@ -7,9 +7,9 @@
 FIELDNAMES = 'unique_identifier,ref_number,vendor_name,contract_date,economic_object_code,description_en,description_fr,contract_period_start,delivery_date,contract_value,original_value,amendment_value,comments_en,comments_fr,additional_comments_en,additional_comments_fr,agreement_type_code,commodity_type_code,commodity_code,country_of_origin,solicitation_procedure_code,limited_tendering_reason_code,derogation_code,aboriginal_business,intellectual_property_code,potential_commercial_exploitation,former_public_servant,standing_offer,standing_offer_number,document_type_code,reporting_period,owner_org,owner_org_title'.split(',')
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
-sys.stdout.write(codecs.BOM_UTF8)
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')
 out_csv.writeheader()
 

bin/migrate/migrate_contracts_2016_08.py

@@ -7,9 +7,9 @@
 FIELDNAMES = 'reference_number,procurement_id,vendor_name,contract_date,economic_object_code,description_en,description_fr,contract_period_start,delivery_date,contract_value,original_value,amendment_value,comments_en,comments_fr,additional_comments_en,additional_comments_fr,agreement_type_code,commodity_type_code,commodity_code,country_of_origin,solicitation_procedure_code,limited_tendering_reason_code,derogation_code,aboriginal_business,intellectual_property_code,potential_commercial_exploitation,former_public_servant,standing_offer,standing_offer_number,document_type_code,reporting_period,owner_org,owner_org_title'.split(',')
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
-sys.stdout.write(codecs.BOM_UTF8)
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')
 out_csv.writeheader()
 

bin/migrate/migrate_contracts_2019_11.py

@@ -19,59 +19,66 @@
 out_csv.writeheader()
 
 
-for line in in_csv:
-    line['vendor_postal_code'] = ''
-    line['buyer_name'] = ''
-    line['contract_value'] = line['contract_value'].replace('$','').replace(',','')
-    line['original_value'] = line['original_value'].replace('$','').replace(',','')
-    line['amendment_value'] = line['amendment_value'].replace('$','').replace(',','')
-    line['trade_agreement'] = ''
-    line['land_claims'] = ''
-    line['commodity_type'] = line.pop('commodity_type_code')
-    line['solicitation_procedure'] = line.pop('solicitation_procedure_code')
-    line['limited_tendering_reason'] = line.pop('limited_tendering_reason_code')
-    line['trade_agreement_exceptions'] = line.pop('exemption_code')
-    line['aboriginal_business_incidental'] = line.pop('aboriginal_business')
-    line['aboriginal_business'] = ''
-    line['intellectual_property'] = line.pop('intellectual_property_code')
-    line['contracting_entity'] = line.pop('standing_offer')
-    line['instrument_type'] = line.pop('document_type_code')
-    line['country_of_vendor'] = line.pop('country_of_origin')
-    line['number_of_bids'] = ''
-    line['article_6_exceptions'] = ''
-    line['award_criteria'] = ''
-    line['socioeconomic_indicator'] = ''
-    line['user_modified'] = '*'  # special "we don't know" value
+try:
+    for line in in_csv:
+        line['vendor_postal_code'] = ''
+        line['buyer_name'] = ''
+        line['contract_value'] = line['contract_value'].replace('$','').replace(',','')
+        line['original_value'] = line['original_value'].replace('$','').replace(',','')
+        line['amendment_value'] = line['amendment_value'].replace('$','').replace(',','')
+        line['trade_agreement'] = ''
+        line['land_claims'] = ''
+        line['commodity_type'] = line.pop('commodity_type_code')
+        line['solicitation_procedure'] = line.pop('solicitation_procedure_code')
+        line['limited_tendering_reason'] = line.pop('limited_tendering_reason_code')
+        line['trade_agreement_exceptions'] = line.pop('exemption_code')
+        line['aboriginal_business_incidental'] = line.pop('aboriginal_business')
+        line['aboriginal_business'] = ''
+        line['intellectual_property'] = line.pop('intellectual_property_code')
+        line['contracting_entity'] = line.pop('standing_offer')
+        line['instrument_type'] = line.pop('document_type_code')
+        line['country_of_vendor'] = line.pop('country_of_origin')
+        line['number_of_bids'] = ''
+        line['article_6_exceptions'] = ''
+        line['award_criteria'] = ''
+        line['socioeconomic_indicator'] = ''
+        line['user_modified'] = '*'  # special "we don't know" value
 
-    # clean up some common mistakes
-    if line['contracting_entity'] == 'PSPCSOSA':  # code changed in 2016!
-        line['contracting_entity'] = 'PWSOSA'
-    if line['contracting_entity'] in ('N/A', 'N', 'NUL'):
-        line['contracting_entity'] = ''
-    line['country_of_vendor'] = line['country_of_vendor'].upper().strip()
-    if line['country_of_vendor'].startswith('CAN'):
-        line['country_of_vendor'] = 'CA'
-    if line['country_of_vendor'].startswith('USA'):
-        line['country_of_vendor'] = 'US'
-    line['instrument_type'] = line['instrument_type'].upper().strip()
-    line['intellectual_property'] = line['intellectual_property'].upper().strip()
-    if ':' in line['intellectual_property']:
-        line['intellectual_property'] = line['intellectual_property'].split(':')[0]
-    if line['intellectual_property'] == 'N/A':
-        line['intellectual_property'] = 'NA'
-    line['commodity_type'] = line['commodity_type'].upper().strip()
-    if line['commodity_type'].startswith('GOOD'):
-        line['commodity_type'] = 'G'
-    if line['commodity_type'].startswith('SERVICE'):
-        line['commodity_type'] = 'S'
-    if ':' in line['commodity_type']:
-        line['commodity_type'] = line['commodity_type'].split(':')[0]
-    line['solicitation_procedure'] = line['solicitation_procedure'].upper().strip()
-    if ':' in line['solicitation_procedure']:
-        line['solicitation_procedure'] = line['solicitation_procedure'].split(':')[0]
-    if line['solicitation_procedure'].startswith('NON-COMPET'):
-        line['solicitation_procedure'] = 'TN'
-    if ':' in line['limited_tendering_reason']:
-        line['limited_tendering_reason'] = line['limited_tendering_reason'].split(':')[0]
+        # clean up some common mistakes
+        if line['contracting_entity'] == 'PSPCSOSA':  # code changed in 2016!
+            line['contracting_entity'] = 'PWSOSA'
+        if line['contracting_entity'] in ('N/A', 'N', 'NUL'):
+            line['contracting_entity'] = ''
+        line['country_of_vendor'] = line['country_of_vendor'].upper().strip()
+        if line['country_of_vendor'].startswith('CAN'):
+            line['country_of_vendor'] = 'CA'
+        if line['country_of_vendor'].startswith('USA'):
+            line['country_of_vendor'] = 'US'
+        line['instrument_type'] = line['instrument_type'].upper().strip()
+        line['intellectual_property'] = line['intellectual_property'].upper().strip()
+        if ':' in line['intellectual_property']:
+            line['intellectual_property'] = line['intellectual_property'].split(':')[0]
+        if line['intellectual_property'] == 'N/A':
+            line['intellectual_property'] = 'NA'
+        line['commodity_type'] = line['commodity_type'].upper().strip()
+        if line['commodity_type'].startswith('GOOD'):
+            line['commodity_type'] = 'G'
+        if line['commodity_type'].startswith('SERVICE'):
+            line['commodity_type'] = 'S'
+        if ':' in line['commodity_type']:
+            line['commodity_type'] = line['commodity_type'].split(':')[0]
+        line['solicitation_procedure'] = line['solicitation_procedure'].upper().strip()
+        if ':' in line['solicitation_procedure']:
+            line['solicitation_procedure'] = line['solicitation_procedure'].split(':')[0]
+        if line['solicitation_procedure'].startswith('NON-COMPET'):
+            line['solicitation_procedure'] = 'TN'
+        if ':' in line['limited_tendering_reason']:
+            line['limited_tendering_reason'] = line['limited_tendering_reason'].split(':')[0]
 
-    out_csv.writerow(line)
+        out_csv.writerow(line)
+
+except KeyError:
+    if 'warehouse' in sys.argv:
+        sys.exit(85)
+    else:
+        raise

bin/migrate/migrate_contracts_nil_2019_11.py

@@ -10,10 +10,15 @@
 sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
+
+if 'warehouse' in sys.argv and 'user_modified' in in_csv.fieldnames:
+    sys.exit(85)
+
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')
 out_csv.writeheader()
 
 for line in in_csv:
-    line['user_modified'] = '*'  # special "we don't know" value
+    if 'warehouse' not in sys.argv:
+        line['user_modified'] = '*'  # special "we don't know" value
 
     out_csv.writerow(line)

bin/migrate/migrate_contractsa_2019_06.py

@@ -8,6 +8,7 @@
 FIELDNAMES = 'year,contract_goods_number_of,contracts_goods_original_value,contracts_goods_amendment_value,contract_service_number_of,contracts_service_original_value,contracts_service_amendment_value,contract_construction_number_of,contracts_construction_original_value,contracts_construction_amendment_value,acquisition_card_transactions_number_of,acquisition_card_transactions_total_value,record_created,record_modified,user_modified,owner_org,owner_org_title'.split(',')
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')

bin/migrate/migrate_grants_2018_04.py

@@ -11,10 +11,10 @@
 FIELDNAMES = 'ref_number,amendment_number,amendment_date,agreement_type,recipient_type,recipient_business_number,recipient_legal_name,recipient_operating_name,research_organization_name,recipient_country,recipient_province,recipient_city,recipient_postal_code,federal_riding_name_en,federal_riding_name_fr,federal_riding_number,prog_name_en,prog_name_fr,prog_purpose_en,prog_purpose_fr,agreement_title_en,agreement_title_fr,agreement_number,agreement_value,foreign_currency_type,foreign_currency_value,agreement_start_date,agreement_end_date,coverage,description_en,description_fr,naics_identifier,expected_results_en,expected_results_fr,additional_information_en,additional_information_fr,record_created,record_modified,user_modified,owner_org,owner_org_title'.split(',')
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')
-sys.stdout.write(codecs.BOM_UTF8)
 out_csv.writeheader()
 
 def norm_date(d):

bin/migrate/migrate_grants_2019_11.py

@@ -5,6 +5,7 @@
 import codecs
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 reader = csv.DictReader(sys.stdin)
 writer = csv.DictWriter(sys.stdout, fieldnames=reader.fieldnames)

bin/migrate/migrate_grants_nil_2018_04.py

@@ -11,6 +11,7 @@
 FIELDNAMES = 'fiscal_year,quarter,record_created,record_modified,user_modified,owner_org,owner_org_title'.split(',')
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')

bin/migrate/migrate_hospitalityq_2016_09.py

@@ -7,10 +7,9 @@
 FIELDNAMES = 'ref_number,name,title_en,title_fr,description_en,description_fr,start_date,end_date,employee_attendees,guest_attendees,location_en,location_fr,total,owner_org,owner_org_title'.split(',')
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
-
-sys.stdout.write(codecs.BOM_UTF8)
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')
 out_csv.writeheader()
 

bin/migrate/migrate_hospitalityq_2019_06.py

@@ -19,7 +19,7 @@
 }
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
-
+sys.stdout.write(codecs.BOM_UTF8)
 
 def norm_date(d, prefer_format):
     "handle some creative thinking about what constitutes a date"
@@ -76,7 +76,7 @@ def error(msg, value=''):
             line['start_date'] = norm_date(
                 line['start_date'],
                 ORG_PREFER_FORMAT.get(line['owner_org']))
-            if line['start_date'] >= datetime(2019, 6, 21):
+            if 'warehouse' not in sys.argv and line['start_date'] >= datetime(2019, 6, 21):
                 error('start_date in the future', line['start_date'])
                 continue
         except ValueError:

bin/migrate/migrate_hospitalityq_nil_2019_06.py

@@ -7,6 +7,7 @@
 FIELDNAMES = 'year,month,record_created,record_modified,user_modified,owner_org,owner_org_title'.split(',')
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')

bin/migrate/migrate_inventory_2017_04.py

@@ -7,6 +7,7 @@
 FIELDNAMES = ['ref_number', 'title_en', 'title_fr', 'description_en', 'description_fr', 'publisher_en', 'publisher_fr', 'date_published', 'language', 'size', 'eligible_for_release', 'program_alignment_architecture_en', 'program_alignment_architecture_fr', 'date_released', 'portal_url_en', 'portal_url_fr', 'user_votes', 'owner_org', 'owner_org_title']
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')

bin/migrate/migrate_reclassification_2020_01.py

@@ -8,6 +8,7 @@
 FIELDNAMES = 'ref_number,job_number,pos_number,date,pos_title_en,pos_title_fr,old_class_group_code,old_class_level,new_class_group_code,new_class_level,old_differential,new_differential,reason_en,reason_fr,record_created,record_modified,user_modified,owner_org,owner_org_title'.split(',')
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')

bin/migrate/migrate_reclassification_nil_2020_01.py

@@ -8,6 +8,7 @@
 FIELDNAMES = 'year,quarter,record_created,record_modified,user_modified,owner_org,owner_org_title'.split(',')
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')

bin/migrate/migrate_service_2018_12.py

@@ -8,6 +8,7 @@
 FIELDNAMES = 'fiscal_yr,service_id,service_name_en,service_name_fr,external_internal,service_type,special_designations,service_description_en,service_description_fr,authority_en,authority_fr,service_url_en,service_url_fr,program_name_en,program_name_fr,program_id_code,client_target_groups,service_fee,cra_business_number,use_of_sin,online_applications,web_visits_info_service,calls_received,in_person_applications,email_applications,fax_applications,postal_mail_applications,e_registration,e_authentication,e_application,e_decision,e_issuance,e_feedback,client_feedback,special_remarks_en,special_remarks_fr,record_created,record_modified,user_modified,owner_org,owner_org_title'.split(',')
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')

bin/migrate/migrate_travela_2018_10.py

@@ -9,6 +9,7 @@
 FIELDNAMES = ['year', 'mandate_description_en', 'mandate_description_fr', 'operational_activities_kdollars', 'key_stakeholders_kdollars', 'training_kdollars', 'other_kdollars', 'internal_governance_kdollars', 'non_public_servants_kdollars', 'public_servants_kdollars', 'hospitality_kdollars', 'conference_fees_kdollars', 'minister_kdollars', 'travel_compared_fiscal_year_en', 'travel_compared_fiscal_year_fr', 'hospitality_compared_fiscal_year_en', 'hospitality_compared_fiscal_year_fr', 'conference_fees_compared_fiscal_year_en', 'conference_fees_compared_fiscal_year_fr', 'minister_compared_fiscal_year_en', 'minister_compared_fiscal_year_fr', 'record_created', 'record_modified', 'user_modified', 'owner_org', 'owner_org_title']
 
 assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
 
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')

bin/migrate/migrate_travelq_2019_06.py

@@ -18,9 +18,6 @@
     'dnd-mdn': '%d/%m/%Y',
 }
 
-assert sys.stdin.read(3) == codecs.BOM_UTF8
-
-
 def norm_date(d, prefer_format):
     "handle some creative thinking about what constitutes a date"
     d = d.replace('.', '-').strip()
@@ -48,6 +45,9 @@ def norm_date(d, prefer_format):
     return from_excel(int(d))
 
 
+assert sys.stdin.read(3) == codecs.BOM_UTF8
+sys.stdout.write(codecs.BOM_UTF8)
+
 in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8')
 out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8')
 out_csv.writeheader()