Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Setup PETL with jobs "diagnosis," "services," and "visit" #28

Merged
merged 18 commits into from
Jul 28, 2021
Merged

Setup PETL with jobs "diagnosis," "services," and "visit" #28

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
986b4c1
petl folder strucutre plus jobs
mhuamani98 Jul 14, 2021
10c2d20
Update source.sql
mhuamani98 Jul 21, 2021
ad2d47f
Update source.sql
mhuamani98 Jul 21, 2021
ab62caa
Update source.sql
mhuamani98 Jul 21, 2021
fdd4f68
Update configuration/pih/petl/jobs/general/diagnosis/source.sql
mhuamani98 Jul 22, 2021
834258c
Update configuration/pih/petl/jobs/general/diagnosis/source.sql
mhuamani98 Jul 22, 2021
56c0f76
Update configuration/pih/petl/jobs/general/diagnosis/source.sql
mhuamani98 Jul 22, 2021
5078ea7
Update configuration/pih/petl/jobs/general/services/source.sql
mhuamani98 Jul 27, 2021
f1f40b9
Added Working days
mhuamani98 Jul 27, 2021
ef74cde
Added working days
mhuamani98 Jul 27, 2021
219658e
Added working days
mhuamani98 Jul 27, 2021
a66ea16
Update source.sql
mhuamani98 Jul 27, 2021
7f6f3b6
Update source.sql
mhuamani98 Jul 27, 2021
7749080
Update source.sql
mhuamani98 Jul 27, 2021
a357d99
Update configuration/pih/petl/jobs/general/diagnosis/source.sql
mhuamani98 Jul 27, 2021
c96a37e
Update source.sql
mhuamani98 Jul 27, 2021
3ebf7b9
Update source.sql
mhuamani98 Jul 27, 2021
8be5ea5
Update source.sql
mhuamani98 Jul 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions configuration/pih/petl/datasources/mysql/openmrs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
databaseType: "mysql"
host: ${mysql.host}
port: ${mysql.port}
databaseName: ${mysql.databaseName}
user: ${mysql.user}
password: ${mysql.password}
options: ${mysql.options}
6 changes: 6 additions & 0 deletions configuration/pih/petl/datasources/sqlserver/openmrs_extractions.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
databaseType: "sqlserver"
host: ${sqlserver.host}
port: ${sqlserver.port}
databaseName: ${sqlserver.databaseName}
user: ${sqlserver.user}
password: ${sqlserver.password}
13 changes: 13 additions & 0 deletions configuration/pih/petl/jobs/general/diagnosis/job.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
type: "sqlserver-bulk-import"
configuration:
extract:
datasource: "mysql/openmrs.yml"
query: "general/diagnosis/source.sql"

load:
datasource: "sqlserver/openmrs_extractions.yml"
table: "consult_diagnosis"
schema: "general/diagnosis/target.sql"

schedule:
cron: "0 30 6 ? * *"
52 changes: 52 additions & 0 deletions configuration/pih/petl/jobs/general/diagnosis/source.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
select encounter_type_id into @tb_general from encounter_type where uuid= 'aa42cc6c-b9ee-4850-926c-dda4bb14d890'; --50 tb intake
select encounter_type_id into @covid_general from encounter_type where uuid='8d50b938-dcf9-4b8e-9938-e625bd2f0a81'; --45 covid-19 admission
select encounter_type_id into @hiv_general from encounter_type where uuid='c31d306a-40c4-11e7-a919-92ebcb67fe33'; --30 hiv intake

drop temporary table if exists temp_diagnosis;
create temporary table temp_diagnosis
(
especialidad varchar(20),
frecuencia_dias int,
frecuencia_semana int,
frecuencia_mes int,
frecuencia_manana int,
frecuencia_tarde int
);

set @YEAR=YEAR(NOW());

insert into temp_diagnosis (especialidad,frecuencia_dias,frecuencia_semana,frecuencia_mes,frecuencia_manana,frecuencia_tarde)
select 'tuberculosis',(count(encounter_type))/251 as frecuencia_dia,
(count(encounter_type)/(WEEK(MAX(encounter_datetime)) - WEEK(MIN(encounter_datetime))+1)) as frecuencia_mes,
mhuamani98 marked this conversation as resolved.
Show resolved Hide resolved
(count(encounter_type)/(MONTH(MAX(encounter_datetime))-MONTH(MIN(encounter_datetime))+1)) as frecuencia_mes,
(select count(encounter_type)/251 from encounter where HOUR (encounter_datetime) BETWEEN 01 AND 11 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 )as frecuencia_manana,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
(select count(encounter_type)/251 from encounter where HOUR (encounter_datetime) BETWEEN 01 AND 11 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 )as frecuencia_manana,
(select count(encounter_type)/251 from encounter where HOUR (encounter_datetime) BETWEEN 00 AND 11 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 )as frecuencia_manana,

no? Para la hora despues de medianoche?

(select count(encounter_type)/251 from encounter where HOUR (encounter_datetime) BETWEEN 12 AND 23 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 ) as frecuencia_tarde
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tal vez queriras excluyir 00:00 y 23:59, que son horas associadas con encuentros sin hora exacto

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Incluyo las 24 horas para contar todos los encuentros que se hayan hecho, ya que por motivos de red algunas citas puede que no se registren ese mismo día en que se realizo el registro y tenga que hacerlo el siguiente.
No se si eso te parece correcto.

from encounter
where encounter_type=@tb_general and YEAR(encounter_datetime) = @YEAR;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

El calculacion que quieres es para este año del calendario?

En este caso, debes adaptar el numero de dias de trabajo (251) para ser el numero aproximo de estos dias que han pasado, no? Algo como DATEDIFF(NOW(), YEAR(NOW) + "-1-1") / 365 * 251?

O quieres calcular por los ultimos 365 dias?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Que hace el "-1-1"(puedes explicarlo en ingles para mayor precisión)?
PD: Gracias por el tip, la idea era al principio esa pero no encontré un modo de aplicarla en el momento..

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

YEAR(NOW) + "-1-1" debe producir 2021-1-1, la fecha.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

¿Se supone que arroja un null en lugar de un decimal o un integer ?. Además, ejecutar YEAR (NOW) +" -1-1 ", muestra solo el año 2020 en lugar del formato de fecha completo y el año correcto. Quizás sería mejor dejarlo con "2021-01-01" por ahora.¿?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh noooo, SQL es un fracaso. Hace CONCAT(YEAR(NOW()), '-01-01')

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All done!


insert into temp_diagnosis (especialidad,frecuencia_dias,frecuencia_semana,frecuencia_mes,frecuencia_manana,frecuencia_tarde)
select 'covid',(count(encounter_type))/251 as frecuencia_dia,
(count(encounter_type)/(WEEK(MAX(encounter_datetime)) - WEEK(MIN(encounter_datetime))+1)) as frecuencia_mes,
mhuamani98 marked this conversation as resolved.
Show resolved Hide resolved
(count(encounter_type)/(MONTH(MAX(encounter_datetime))-MONTH(MIN(encounter_datetime))+1)) as frecuencia_mes,
(select count(encounter_type)/251 from encounter where HOUR (encounter_datetime) BETWEEN 01 AND 11 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 )as frecuencia_manana,
(select count(encounter_type)/251 from encounter where HOUR (encounter_datetime) BETWEEN 12 AND 23 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 ) as frecuencia_tarde
from encounter
where encounter_type=@covid_general and YEAR(encounter_datetime) = @YEAR;

insert into temp_diagnosis (especialidad,frecuencia_dias,frecuencia_semana,frecuencia_mes,frecuencia_manana,frecuencia_tarde)
select 'hiv',(count(encounter_type))/251 as frecuencia_dia,
(count(encounter_type)/(WEEK(MAX(encounter_datetime)) - WEEK(MIN(encounter_datetime))+1)) as frecuencia_mes,
mhuamani98 marked this conversation as resolved.
Show resolved Hide resolved
(count(encounter_type)/(MONTH(MAX(encounter_datetime))-MONTH(MIN(encounter_datetime))+1)) as frecuencia_mes,
(select count(encounter_type)/251 from encounter where HOUR (encounter_datetime) BETWEEN 01 AND 11 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 )as frecuencia_manana,
(select count(encounter_type)/251 from encounter where HOUR (encounter_datetime) BETWEEN 12 AND 23 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 ) as frecuencia_tarde
from encounter
where encounter_type=@hiv_general and YEAR(encounter_datetime) = @YEAR;

select
especialidad,
frecuencia_dias,
frecuencia_semana,
frecuencia_mes,
frecuencia_manana,
frecuencia_tarde
from temp_diagnosis;
9 changes: 9 additions & 0 deletions configuration/pih/petl/jobs/general/diagnosis/target.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
create table consult_diagnosis
(
especialidad varchar(20),
frecuencia_dias int,
frecuencia_semana int,
frecuencia_mes int,
frecuencia_manana int,
frecuencia_tarde int
);
13 changes: 13 additions & 0 deletions configuration/pih/petl/jobs/general/services/job.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
type: "sqlserver-bulk-import"
configuration:
extract:
datasource: "mysql/openmrs.yml"
query: "general/services/source.sql"

load:
datasource: "sqlserver/openmrs_extractions.yml"
table: "consult_services"
schema: "general/services/target.sql"

schedule:
cron: "0 30 6 ? * *"
33 changes: 33 additions & 0 deletions configuration/pih/petl/jobs/general/services/source.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
select encounter_type_id into @examenes from encounter_type where uuid= 'b3a0e3ad-b80c-4f3f-9626-ace1ced7e2dd'; --26 test orders(lab)

drop temporary table if exists temp_services;
create temporary table temp_services
(
especialidad varchar(20),
frecuencia_dias int,
frecuencia_semana int,
frecuencia_mes int,
frecuencia_manana int,
frecuencia_tarde int
);


set @YEAR=YEAR(NOW());

insert into temp_services (especialidad,frecuencia_dias,frecuencia_semana,frecuencia_mes,frecuencia_manana,frecuencia_tarde)
select 'examenes',(count(encounter_type))/251 as frecuencia_dia,
(count(encounter_type)/(WEEK(MAX(encounter_datetime)) - WEEK(MIN(encounter_datetime))+1)) as frecuencia_mes,
mhuamani98 marked this conversation as resolved.
Show resolved Hide resolved
(count(encounter_type)/(MONTH(MAX(encounter_datetime))-MONTH(MIN(encounter_datetime))+1)) as frecuencia_mes,
(select count(encounter_type)/251 from encounter where HOUR (encounter_datetime) BETWEEN 01 AND 11 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 )as frecuencia_manana,
(select count(encounter_type)/251 from encounter where HOUR (encounter_datetime) BETWEEN 12 AND 23 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 ) as frecuencia_tarde
from encounter
where encounter_type=@examenes and YEAR(encounter_datetime) = @YEAR;

select
especialidad,
frecuencia_dias,
frecuencia_semana,
frecuencia_mes,
frecuencia_manana,
frecuencia_tarde
from temp_services;
9 changes: 9 additions & 0 deletions configuration/pih/petl/jobs/general/services/target.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
create table consult_services
(
especialidad varchar(20),
frecuencia_dias int,
frecuencia_semana int,
frecuencia_mes int,
frecuencia_manana int,
frecuencia_tarde int
);
13 changes: 13 additions & 0 deletions configuration/pih/petl/jobs/general/visit/job.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
type: "sqlserver-bulk-import"
configuration:
extract:
datasource: "mysql/openmrs.yml"
query: "general/visit/source.sql"

load:
datasource: "sqlserver/openmrs_extractions.yml"
table: "consult_visits"
schema: "general/visit/target.sql"

schedule:
cron: "0 30 6 ? * *"
53 changes: 53 additions & 0 deletions configuration/pih/petl/jobs/general/visit/source.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
select encounter_type_id into @tb_general from encounter_type where uuid= 'aa42cc6c-b9ee-4850-926c-dda4bb14d890'; --50 tb intake
select encounter_type_id into @covid_general from encounter_type where uuid='8d50b938-dcf9-4b8e-9938-e625bd2f0a81'; --45 covid-19 admission
select encounter_type_id into @hiv_general from encounter_type where uuid='c31d306a-40c4-11e7-a919-92ebcb67fe33'; --29 hiv intake

drop temporary table if exists temp_consultations;
create temporary table temp_consultations
(
especialidad varchar(20),
frecuencia_dias int,
frecuencia_semana int,
frecuencia_mes int,
frecuencia_manana int,
frecuencia_tarde int
);

set @YEAR=YEAR(NOW());

insert into temp_consultations (especialidad,frecuencia_dias,frecuencia_semana,frecuencia_mes,frecuencia_manana,frecuencia_tarde)
select 'tuberculosis',(count(encounter_type))/251 as frecuencia_dia,
(count(encounter_type)/(WEEK(MAX(encounter_datetime)) - WEEK(MIN(encounter_datetime))+1)) as frecuencia_mes,
(count(encounter_type)/(MONTH(MAX(encounter_datetime))-MONTH(MIN(encounter_datetime))+1)) as frecuencia_mes,
(select count(encounter_type)/251 from encounter where HOUR (encounter_datetime) BETWEEN 01 AND 11 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 )as frecuencia_manana,
(select count(encounter_type)/251 from encounter where HOUR (encounter_datetime) BETWEEN 12 AND 23 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 ) as frecuencia_tarde
from encounter
where encounter_type=@tb_general and YEAR(encounter_datetime) = @YEAR;

insert into temp_consultations (especialidad,frecuencia_dias,frecuencia_semana,frecuencia_mes,frecuencia_manana,frecuencia_tarde)
select 'covid',(count(encounter_type))/251 as frecuencia_dia,
(count(encounter_type)/(WEEK(MAX(encounter_datetime)) - WEEK(MIN(encounter_datetime))+1)) as frecuencia_mes,
(count(encounter_type)/(MONTH(MAX(encounter_datetime))-MONTH(MIN(encounter_datetime))+1)) as frecuencia_mes,
(select count(encounter_type)/251 from encounter where HOUR (encounter_datetime) BETWEEN 01 AND 11 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 )as frecuencia_manana,
(select count(encounter_type)/251 from encounter where HOUR (encounter_datetime) BETWEEN 12 AND 23 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 ) as frecuencia_tarde
from encounter
where encounter_type=@covid_general and YEAR(encounter_datetime) = @YEAR;

insert into temp_consultations (especialidad,frecuencia_dias,frecuencia_semana,frecuencia_mes,frecuencia_manana,frecuencia_tarde)
select 'hiv',(count(encounter_type))/251 as frecuencia_dia,
(count(encounter_type)/(WEEK(MAX(encounter_datetime)) - WEEK(MIN(encounter_datetime))+1)) as frecuencia_mes,
(count(encounter_type)/(MONTH(MAX(encounter_datetime))-MONTH(MIN(encounter_datetime))+1)) as frecuencia_mes,
(select count(encounter_type)/251 from encounter where HOUR (encounter_datetime) BETWEEN 01 AND 11 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 )as frecuencia_manana,
(select count(encounter_type)/251 from encounter where HOUR (encounter_datetime) BETWEEN 12 AND 23 AND MINUTE (encounter_datetime) BETWEEN 00 AND 59 ) as frecuencia_tarde
from encounter
where encounter_type=@hiv_general and YEAR(encounter_datetime) = @YEAR;

select
especialidad,
frecuencia_dias,
frecuencia_semana,
frecuencia_mes,
frecuencia_manana,
frecuencia_tarde
from temp_consultations;

9 changes: 9 additions & 0 deletions configuration/pih/petl/jobs/general/visit/target.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
create table consult_visits
(
especialidad varchar(50),
frecuencia_dias int,
frecuencia_semana int,
frecuencia_mes int,
frecuencia_manana int,
frecuencia_tarde int
);