WIP: Added MySQL (and MariaDB) source.

Gemfile.lock

@@ -7,6 +7,7 @@ PATH
       grape (~> 2.0)
       iteraptor (~> 0.10)
       mongo (~> 2.18)
+      mysql2 (~> 0.5)
       rack (~> 3.0)
       rackup (~> 2.1)
       sequel (~> 5.68)
@@ -78,6 +79,7 @@ GEM
     mustermann-grape (1.1.0)
       mustermann (>= 1.0.0)
     mutex_m (0.2.0)
+    mysql2 (0.5.6)
     net-http (0.4.1)
       uri
     parallel (1.26.1)

cmr-entity-resolution.gemspec

@@ -32,6 +32,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'grape', '~> 2.0'
   s.add_dependency 'iteraptor', '~> 0.10'
   s.add_dependency 'mongo', '~> 2.18'
+  s.add_dependency 'mysql2', '~> 0.5'
   s.add_dependency 'rack', '~> 3.0'
   s.add_dependency 'rackup', '~> 2.1'
   s.add_dependency 'sequel', '~> 5.68'

docs/examples/assets/config.mariadb.yml

@@ -0,0 +1,72 @@
+log_level: debug
+match_level: 2
+match_score: 5
+filters:
+  - NonHuman
+  - filter: ValueIs
+    field: TYPE
+    value: DEF
+    inverse: false
+
+senzing:
+  host: api
+  port: 8250
+  data_source: PEOPLE
+  tls: false
+
+sources:
+  mariadb:
+    type: MySQL
+    host: maraidb
+#    host: 127.0.0.1
+    database: people
+    table: people
+    username: root
+    password: password
+    field_map:
+      party_id: OTHER_ID_PARTY
+      last_name: PRIMARY_NAME_LAST
+      first_name: PRIMARY_NAME_FIRST
+      gender: GENDER
+      birth_date: DATE_OF_BIRTH
+      dr_lic_num: DRIVERS_LICENSE_NUMBER
+      dr_lic_state: DRIVERS_LICENSE_STATE
+      ssn: SSN_NUMBER
+      address_1: HOME_ADDR_LINE1
+      address_2: HOME_ADDR_LINE2
+      city: HOME_ADDR_CITY
+      state_code: HOME_ADDR_STATE
+      zip_code: HOME_ADDR_POSTAL_CODE
+      bus_phone: WORK_PHONE_NUMBER
+      home_phone: CELL_PHONE_NUMBER
+      email_address: EMAIL_ADDRESS
+      otn: OTHER_ID_NUMBER
+      party_code: TYPE
+
+destination:
+  type: CSV
+  path: /etc/cmr/export/export.csv
+  overwrite: true
+  headers:
+    - person_id
+    - database
+    - party_id
+    - match_score
+    - potential_person_id
+    - potential_match_score
+  field_map:
+    ENTITY_ID: person_id
+    DATABASE: database
+    PARTY_ID: party_id
+    MATCH_SCORE: match_score
+    RELATED_RECORD_ID: potential_person_id
+    RELATED_MATCH_SCORE: potential_match_score
+  # TODO: Remove this option once we can export via the API.
+  export_file: /etc/cmr/export/export.json
+  transformations:
+    - transform: SplitValue
+      field: RECORD_ID
+      delimiter: "-"
+      parts:
+        0: DATABASE
+        1: PARTY_ID

docs/examples/assets/docker-compose.mariadb.yml

@@ -0,0 +1,27 @@
+services:
+  maraidb:
+    image: mariadb
+    environment:
+      # See https://hub.docker.com/_/mariadb
+      MARIADB_ROOT_PASSWORD: ${INFORMIX_PASSWORD:-password}
+      MARIADB_DATABASE: people
+    networks:
+      - senzing
+    ports:
+      - 3306:3306
+    restart: always
+    volumes:
+      - maraidb:/var/lib/mysql
+      - type: bind
+        source: ./docs/examples/assets/mysql-schema.sql
+        target: /docker-entrypoint-initdb.d/schema.sql
+      - type: bind
+        source: ./docs/examples/assets/import.csv
+        target: /docker-entrypoint-initdb.d/import.csv
+
+networks:
+  senzing:
+    name: ${SENZING_DOCKER_NETWORK:-senzing-network}
+
+volumes:
+  maraidb:

docs/examples/assets/mysql-schema.sql

@@ -0,0 +1,27 @@
+DROP TABLE IF EXISTS people;
+CREATE TABLE people(
+   party_id VARCHAR(255) NOT NULL,
+   last_name VARCHAR(255) NULL,
+   first_name VARCHAR(255) NULL,
+   gender VARCHAR(255) NULL,
+   birth_date VARCHAR(255) NULL,
+   dr_lic_num VARCHAR(255) NULL,
+   dr_lic_state VARCHAR(255) NULL,
+   ssn VARCHAR(255) NULL,
+   address_1 VARCHAR(255) NULL,
+   address_2 VARCHAR(255) NULL,
+   city VARCHAR(255) NULL,
+   state_code VARCHAR(255) NULL,
+   zip_code VARCHAR(255) NULL,
+   bus_phone VARCHAR(255) NULL,
+   home_phone VARCHAR(255) NULL,
+   email_address VARCHAR(255) NULL,
+   otn VARCHAR(255) NULL,
+   party_code VARCHAR(255) NULL
+);
+
+LOAD DATA LOCAL INFILE "/docker-entrypoint-initdb.d/import.csv"
+     INTO TABLE people
+     FIELDS TERMINATED BY ','
+     LINES TERMINATED BY '\n'
+    IGNORE 1 ROWS;

lib/source.rb

@@ -3,6 +3,7 @@
 require_relative 'source/api'
 require_relative 'source/csv'
 require_relative 'source/informix'
+require_relative 'source/mysql'
 
 # Helper methods for loading sources.
 module Source

lib/source/mysql.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'csv'
+require 'sequel'
+require_relative 'file'
+
+module Source
+  # MySQL source for data imports.
+  class MySQL < Base
+    def each
+      table = db[@source_config[:table].to_sym]
+      table.each do |record|
+        record.transform_keys! { |key| field_mapper(key) }
+        yield record
+      end
+    end
+
+    private
+
+    # Establishes a database connection and proxies calls to the database.
+    #
+    # @return [Sequel::Database]
+    def db
+      @db ||= Sequel.connect(
+        adapter: 'mysql2',
+        host: @source_config[:host],
+        database: @source_config[:database],
+        port: @source_config[:port],
+        user: @source_config[:username],
+        password: @source_config[:password],
+        schema: @source_config[:schema] || @source_config[:username],
+        security: @source_config[:security]
+      )
+    end
+
+    def defaults
+      super.merge({ port: 3306, security: nil })
+    end
+  end
+end