Fix: Validate VIN & allow vehicles with no transmission

Uses some fancy SQL to remove NULL entries for transmissions when at least one row has a value for the transmission, so that both vehicles with no transmission and with transmissions can properly be decoded. Also adds a few "tests" to validate this.

Author: spnda

README.md

@@ -1,6 +1,6 @@
 # volvo_vida_db
 
-This repository contains scripts to extract and work with data from the internal Volvo VIDA database.
+This repository contains scripts to extract and work with data from the internal Volvo VIDA 2014D database.
 
 Some SQL statements are based on procedures found in the VIDA SQL Server database and from scripts in [Tigo2000's repository](https://github.com/Tigo2000/Volvo-VIDA/).
 

scripts/test_vin_decoder.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+from vin_decoder import PartnerGroup, decode_vin
+from read_csv import DatabaseFile, get_csvs
+
+# These are some VINs I found in the VIDA source code which are used for testing I think.
+# Entering x0000000000000000 where x is any digit into the VIN input in VIDA will transform into
+# the i-ths VIN in the below list. The last two VINs seem to be left-over from testing code in VIDA.
+test_vins = [("YV1LZ56DXY2736349", PartnerGroup.AME),
+            ("YV1LW5642W2360395", PartnerGroup.EUR),
+            ("YV1945866W1234629", PartnerGroup.EUR),
+            ("YV1LS61F2Y2667227", PartnerGroup.EUR),
+            ("YV1SW61P212048331", PartnerGroup.EUR),
+            # "YV1LW65F2Y683325", <-- Found in VIDA but actually invalid.
+            ("YV1RS65P212000500", PartnerGroup.EUR),
+            ("YV1RS65P922122056", PartnerGroup.EUR),
+            # "YV1TS61R91193741", <-- Also not a valid VIN
+            ("YV1RS59G242370046", PartnerGroup.EUR),
+            ("YV1TS61F2Y1065654", PartnerGroup.EUR),
+            ("YV1KS960XV1104523", PartnerGroup.AME)]
+
+get_csvs(DatabaseFile.vehicle_model, DatabaseFile.model_year, DatabaseFile.partner_group, DatabaseFile.body_style, DatabaseFile.engine, DatabaseFile.transmission)
+for vin, partner in test_vins:
+    try:
+        vehicle = decode_vin(vin, partner_id=partner, cached=False)
+        print(vehicle.get_value_description("vehicle_model"))
+        #vehicle.print()
+    except ValueError as e:
+        print("Failed lookup: " + str(e))

scripts/vin_decoder.py

@@ -72,6 +72,13 @@ def print(self):
         print(f'Transmission: {self.get_value_description("transmission")} [{self.transmission}]')
         print(f'VIDA Profiles: {self.get_vehicle_profiles()}')
 
+def validate_vin(vin: str) -> bool:
+    vin = vin.strip()
+    if len(vin) != 17:
+        return False
+    chassis_no = vin[11:]
+    return chassis_no.isdigit()
+
 # TODO: Don't default to PartnerGroup.EUR but have some detection or add it as a required input for all callers.
 def decode_vin(vin: str, partner_id: PartnerGroup = PartnerGroup.EUR, cached: bool = True) -> Vehicle:
     """
@@ -80,9 +87,11 @@ def decode_vin(vin: str, partner_id: PartnerGroup = PartnerGroup.EUR, cached: bo
     There's more information inside the VIN number, which is not decoded here. You can find more information here:
     https://en.wikibooks.org/wiki/Vehicle_Identification_Numbers_(VIN_codes)/Volvo/VIN_Codes#Position_1_-_3:_World_Manufacturer_Identifier
     """
+    if not validate_vin(vin):
+        raise ValueError('VIN string is not a valid VIN')
+    
     if vin[:3] != 'YV1':
-        print('VIN is not for a Volvo vehicle')
-        return
+        raise ValueError('VIN is not for a Volvo vehicle')
 
     if cached and os.path.exists(f"cache/{vin}.p"):
         # This import is to avoid namespace problems, making the class always be vin_decoder.Vehicle instead of __main__.Vehicle
@@ -109,33 +118,35 @@ def decode_vin(vin: str, partner_id: PartnerGroup = PartnerGroup.EUR, cached: bo
         AND VM.fkPartnerGroup = {partner_id.value}
     """)
 
-    get_csv(DatabaseFile.vehicle_profile)
-
     # For cases where the VIN represents multiple combinations of engines and transmissions we match the table with itself to create
-    # rows with every possible combination of engines and transmissions. However, we also want to allow VINs that only represent a single engine.
+    # rows with every possible combination of engines and transmissions.
+    # However, we also want to allow VINs where the transmission is not determinable or where no transmission is specified in the database.
     combined = duckdb.sql("""
         SELECT DISTINCT c1.fkVehicleModel, c1.fkModelYear, c1.fkPartnerGroup, c1.fkBodyStyle, c1.fkEngine, c2.fkTransmission FROM components AS c1, components AS c2
-        WHERE (c1.fkEngine IS NOT NULL AND c2.fkTransmission IS NOT NULL) OR c1.fkEngine IS NOT NULL
+        WHERE (c1.fkEngine IS NOT NULL AND c2.fkTransmission IS NOT NULL) OR (c1.fkEngine IS NOT NULL AND c2.fkTransmission IS NULL AND NOT EXISTS (
+            SELECT 1 FROM components WHERE fkTransmission IS NOT NULL
+        ))
     """).df()
 
     # Filter all the engine/transmission combinations for actually valid ones from the VehicleProfile table if more than one exists
     if len(combined) > 1:
+        get_csv(DatabaseFile.vehicle_profile)
         filtered = duckdb.sql("""
             SELECT DISTINCT combined.* FROM combined
             INNER JOIN vehicle_profile vp on vp.fkVehicleModel=combined.fkVehicleModel AND vp.fkModelYear=combined.fkModelYear
-            WHERE combined.fkEngine=vp.fkEngine AND combined.fkTransmission=vp.fkTransmission
+            WHERE (combined.fkEngine=vp.fkEngine AND combined.fkTransmission=vp.fkTransmission) OR combined.fkTransmission IS NULL
         """).df()
         combined = filtered
 
     # Replace possible NaN values with 'None' to avoid having float64 columns, and cast everything to int to not use numpy types
     combined = combined.replace({math.nan: 0}).astype('int64')
-    print(combined)
 
     if combined.empty:
-        raise ValueError('Failed to find valid vehicle profiles for VIN')
-    if len(combined) > 1:
-        raise ValueError('More than 1 vehicle profile appeared for VIN')
-
+        raise ValueError('Failed to find valid vehicles for VIN')
+    if len(combined['fkVehicleModel'].unique()) > 1:
+        # Only the vehicle model has to be unique. All other values can technically be unspecified by the VIN.
+        raise ValueError('More than 1 vehicle appeared for VIN')
+    
     from vin_decoder import Vehicle
     vehicle = Vehicle(vin, combined.loc[0])
     if cached: