Setup PETL with jobs "diagnosis," "services," and "visit" (#28)

Co-authored-by: Brandon Istenes <bistenes@gmail.com>

configuration/pih/petl/datasources/mysql/openmrs.yml

@@ -0,0 +1,7 @@
+databaseType: "mysql"
+host: ${mysql.host}
+port: ${mysql.port}
+databaseName: ${mysql.databaseName}
+user: ${mysql.user}
+password: ${mysql.password}
+options: ${mysql.options}

configuration/pih/petl/datasources/sqlserver/openmrs_extractions.yml

@@ -0,0 +1,6 @@
+databaseType: "sqlserver"
+host: ${sqlserver.host}
+port: ${sqlserver.port}
+databaseName: ${sqlserver.databaseName}
+user: ${sqlserver.user}
+password: ${sqlserver.password}

configuration/pih/petl/jobs/general/diagnosis/job.yml

@@ -0,0 +1,13 @@
+type: "sqlserver-bulk-import"
+configuration:
+  extract:
+    datasource: "mysql/openmrs.yml"
+    query:  "general/diagnosis/source.sql"
+
+  load:
+    datasource: "sqlserver/openmrs_extractions.yml"
+    table: "consult_diagnosis"
+    schema: "general/diagnosis/target.sql"
+
+schedule:
+  cron: "0 30 6 ? * *"

configuration/pih/petl/jobs/general/diagnosis/source.sql

@@ -0,0 +1,53 @@
+select encounter_type_id into @tb_general from encounter_type where uuid= 'aa42cc6c-b9ee-4850-926c-dda4bb14d890'; --50 tb intake
+select encounter_type_id into @covid_general from encounter_type where uuid='8d50b938-dcf9-4b8e-9938-e625bd2f0a81'; --45 covid-19 admission
+select encounter_type_id into @hiv_general from encounter_type where uuid='c31d306a-40c4-11e7-a919-92ebcb67fe33'; --30 hiv intake
+
+drop temporary table if exists temp_diagnosis;
+create temporary table temp_diagnosis
+(
+especialidad  varchar(20),
+frecuencia_dias int,
+frecuencia_semana int,
+frecuencia_mes int,
+frecuencia_manana int,
+frecuencia_tarde int
+);
+
+set @YEAR=YEAR(NOW());
+set @diaslaborales=datediff(NOW(),CONCAT(YEAR(NOW()), '-01-01'))/365*251;
+
+insert into temp_diagnosis (especialidad,frecuencia_dias,frecuencia_semana,frecuencia_mes,frecuencia_manana,frecuencia_tarde)
+select 'tuberculosis',(count(encounter_type))/@diaslaborales as frecuencia_dia,
+(count(encounter_type)/(WEEK(MAX(encounter_datetime)) - WEEK(MIN(encounter_datetime))+1)) as frecuencia_semana,
+(count(encounter_type)/(MONTH(MAX(encounter_datetime))-MONTH(MIN(encounter_datetime))+1)) as frecuencia_mes,
+(select count(encounter_type)/@diaslaborales from encounter where HOUR (encounter_datetime) BETWEEN 00 AND 11 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 )as frecuencia_manana,
+(select count(encounter_type)/@diaslaborales from encounter where HOUR (encounter_datetime) BETWEEN 12 AND 23 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 ) as frecuencia_tarde
+from encounter
+where encounter_type=@tb_general and YEAR(encounter_datetime) = @YEAR;
+
+insert into temp_diagnosis (especialidad,frecuencia_dias,frecuencia_semana,frecuencia_mes,frecuencia_manana,frecuencia_tarde)
+select 'covid',(count(encounter_type))/@diaslaborales as frecuencia_dia,
+(count(encounter_type)/(WEEK(MAX(encounter_datetime)) - WEEK(MIN(encounter_datetime))+1)) as frecuencia_semana,
+(count(encounter_type)/(MONTH(MAX(encounter_datetime))-MONTH(MIN(encounter_datetime))+1)) as frecuencia_mes,
+(select count(encounter_type)/@diaslaborales from encounter where HOUR (encounter_datetime) BETWEEN 00 AND 11 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 )as frecuencia_manana,
+(select count(encounter_type)/@diaslaborales from encounter where HOUR (encounter_datetime) BETWEEN 12 AND 23 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 ) as frecuencia_tarde
+from encounter
+where encounter_type=@covid_general and YEAR(encounter_datetime) = @YEAR;
+
+insert into temp_diagnosis (especialidad,frecuencia_dias,frecuencia_semana,frecuencia_mes,frecuencia_manana,frecuencia_tarde)
+select 'hiv',(count(encounter_type))/@diaslaborales as frecuencia_dia,
+(count(encounter_type)/(WEEK(MAX(encounter_datetime)) - WEEK(MIN(encounter_datetime))+1)) as frecuencia_semana,
+(count(encounter_type)/(MONTH(MAX(encounter_datetime))-MONTH(MIN(encounter_datetime))+1)) as frecuencia_mes,
+(select count(encounter_type)/@diaslaborales from encounter where HOUR (encounter_datetime) BETWEEN 00 AND 11 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 )as frecuencia_manana,
+(select count(encounter_type)/@diaslaborales from encounter where HOUR (encounter_datetime) BETWEEN 12 AND 23 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 ) as frecuencia_tarde
+from encounter
+where encounter_type=@hiv_general and YEAR(encounter_datetime) = @YEAR;
+
+select 
+especialidad,
+frecuencia_dias,
+frecuencia_semana,
+frecuencia_mes,
+frecuencia_manana,
+frecuencia_tarde
+from temp_diagnosis;

configuration/pih/petl/jobs/general/diagnosis/target.sql

@@ -0,0 +1,9 @@
+create table consult_diagnosis
+(
+especialidad  varchar(20),
+frecuencia_dias int,
+frecuencia_semana int,
+frecuencia_mes int,
+frecuencia_manana int,
+frecuencia_tarde int
+);

configuration/pih/petl/jobs/general/services/job.yml

@@ -0,0 +1,13 @@
+type: "sqlserver-bulk-import"
+configuration:
+  extract:
+    datasource: "mysql/openmrs.yml"
+    query:  "general/services/source.sql"
+
+  load:
+    datasource: "sqlserver/openmrs_extractions.yml"
+    table: "consult_services"
+    schema: "general/services/target.sql"
+
+schedule:
+  cron: "0 30 6 ? * *"

configuration/pih/petl/jobs/general/services/source.sql

@@ -0,0 +1,34 @@
+select encounter_type_id into @examenes from encounter_type where uuid= 'b3a0e3ad-b80c-4f3f-9626-ace1ced7e2dd'; --26 test orders(lab)
+
+drop temporary table if exists temp_services;
+create temporary table temp_services
+(
+especialidad  varchar(20),
+frecuencia_dias int,
+frecuencia_semana int,
+frecuencia_mes int,
+frecuencia_manana int,
+frecuencia_tarde int
+);
+
+
+set @YEAR=YEAR(NOW());
+set @diaslaborales=datediff(NOW(),CONCAT(YEAR(NOW()), '-01-01'))/365*251;
+
+insert into temp_services (especialidad,frecuencia_dias,frecuencia_semana,frecuencia_mes,frecuencia_manana,frecuencia_tarde)
+select 'examenes',(count(encounter_type))/@diaslaborales as frecuencia_dia,
+(count(encounter_type)/(WEEK(MAX(encounter_datetime)) - WEEK(MIN(encounter_datetime))+1)) as frecuencia_semana,
+(count(encounter_type)/(MONTH(MAX(encounter_datetime))-MONTH(MIN(encounter_datetime))+1)) as frecuencia_mes,
+(select count(encounter_type)/@diaslaborales from encounter where HOUR (encounter_datetime) BETWEEN 00 AND 11 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 )as frecuencia_manana,
+(select count(encounter_type)/@diaslaborales from encounter where HOUR (encounter_datetime) BETWEEN 12 AND 23 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 ) as frecuencia_tarde
+from encounter
+where encounter_type=@examenes and YEAR(encounter_datetime) = @YEAR;
+
+select 
+especialidad,
+frecuencia_dias,
+frecuencia_semana,
+frecuencia_mes,
+frecuencia_manana,
+frecuencia_tarde
+from temp_services;

configuration/pih/petl/jobs/general/services/target.sql

@@ -0,0 +1,9 @@
+create table consult_services
+(
+especialidad  varchar(20),
+frecuencia_dias int,
+frecuencia_semana int,
+frecuencia_mes int,
+frecuencia_manana int,
+frecuencia_tarde int
+);

configuration/pih/petl/jobs/general/visit/job.yml

@@ -0,0 +1,13 @@
+type: "sqlserver-bulk-import"
+configuration:
+  extract:
+    datasource: "mysql/openmrs.yml"
+    query:  "general/visit/source.sql"
+
+  load:
+    datasource: "sqlserver/openmrs_extractions.yml"
+    table: "consult_visits"
+    schema: "general/visit/target.sql"
+
+schedule:
+  cron: "0 30 6 ? * *"

configuration/pih/petl/jobs/general/visit/source.sql

@@ -0,0 +1,54 @@
+select encounter_type_id into @tb_general from encounter_type where uuid= 'aa42cc6c-b9ee-4850-926c-dda4bb14d890'; --50 tb intake
+select encounter_type_id into @covid_general from encounter_type where uuid='8d50b938-dcf9-4b8e-9938-e625bd2f0a81'; --45 covid-19 admission
+select encounter_type_id into @hiv_general from encounter_type where uuid='c31d306a-40c4-11e7-a919-92ebcb67fe33'; --29 hiv intake
+
+drop temporary table if exists temp_consultations;
+create temporary table temp_consultations
+(
+especialidad  varchar(20),
+frecuencia_dias int,
+frecuencia_semana int,
+frecuencia_mes int,
+frecuencia_manana int,
+frecuencia_tarde int
+);
+
+set @YEAR=YEAR(NOW());
+set @diaslaborales=datediff(NOW(),CONCAT(YEAR(NOW()), '-01-01'))/365*251;
+
+insert into temp_consultations (especialidad,frecuencia_dias,frecuencia_semana,frecuencia_mes,frecuencia_manana,frecuencia_tarde)
+select 'tuberculosis',(count(encounter_type))/@diaslaborales as frecuencia_dia,
+(count(encounter_type)/(WEEK(MAX(encounter_datetime)) - WEEK(MIN(encounter_datetime))+1)) as frecuencia_mes,
+(count(encounter_type)/(MONTH(MAX(encounter_datetime))-MONTH(MIN(encounter_datetime))+1)) as frecuencia_mes,
+(select count(encounter_type)/@diaslaborales from encounter where HOUR (encounter_datetime) BETWEEN 00 AND 11 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 )as frecuencia_manana,
+(select count(encounter_type)/@diaslaborales from encounter where HOUR (encounter_datetime) BETWEEN 12 AND 23 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 ) as frecuencia_tarde
+from encounter
+where encounter_type=@tb_general and YEAR(encounter_datetime) = @YEAR;
+
+insert into temp_consultations (especialidad,frecuencia_dias,frecuencia_semana,frecuencia_mes,frecuencia_manana,frecuencia_tarde)
+select 'covid',(count(encounter_type))/@diaslaborales as frecuencia_dia,
+(count(encounter_type)/(WEEK(MAX(encounter_datetime)) - WEEK(MIN(encounter_datetime))+1)) as frecuencia_mes,
+(count(encounter_type)/(MONTH(MAX(encounter_datetime))-MONTH(MIN(encounter_datetime))+1)) as frecuencia_mes,
+(select count(encounter_type)/@diaslaborales from encounter where HOUR (encounter_datetime) BETWEEN 00 AND 11 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 )as frecuencia_manana,
+(select count(encounter_type)/@diaslaborales from encounter where HOUR (encounter_datetime) BETWEEN 12 AND 23 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 ) as frecuencia_tarde
+from encounter
+where encounter_type=@covid_general and YEAR(encounter_datetime) = @YEAR;
+
+insert into temp_consultations (especialidad,frecuencia_dias,frecuencia_semana,frecuencia_mes,frecuencia_manana,frecuencia_tarde)
+select 'hiv',(count(encounter_type))/@diaslaborales as frecuencia_dia,
+(count(encounter_type)/(WEEK(MAX(encounter_datetime)) - WEEK(MIN(encounter_datetime))+1)) as frecuencia_mes,
+(count(encounter_type)/(MONTH(MAX(encounter_datetime))-MONTH(MIN(encounter_datetime))+1)) as frecuencia_mes,
+(select count(encounter_type)/@diaslaborales from encounter where HOUR (encounter_datetime) BETWEEN 00 AND 11 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 )as frecuencia_manana,
+(select count(encounter_type)/@diaslaborales from encounter where HOUR (encounter_datetime) BETWEEN 12 AND 23 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 ) as frecuencia_tarde
+from encounter
+where encounter_type=@hiv_general and YEAR(encounter_datetime) = @YEAR;
+
+select 
+especialidad,
+frecuencia_dias,
+frecuencia_semana,
+frecuencia_mes,
+frecuencia_manana,
+frecuencia_tarde
+from temp_consultations;
+

configuration/pih/petl/jobs/general/visit/target.sql

@@ -0,0 +1,9 @@
+create table consult_visits
+(
+especialidad  varchar(50),
+frecuencia_dias int,
+frecuencia_semana int,
+frecuencia_mes int,
+frecuencia_manana int,
+frecuencia_tarde int
+);