Skip to content

Commit 549c7f3

Browse files
ChakshuGautamChakshuGautam
committed
Bug fixes on the data ingestion front
1 parent 7c6fda5 commit 549c7f3

19 files changed

+5670
-5606
lines changed

‎.DS_Store

6 KB
Binary file not shown.

‎.vscode/settings.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
22
"editor.detectIndentation": false,
3-
"editor.tabSize": 2
3+
"editor.tabSize": 2,
44
}

‎docs/csv-adapter/lexer.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
### Examples
2+
- https://github.com/apache/commons-csv/blob/master/src/main/java/org/apache/commons/csv/CSVFormat.java

‎docs/grade, attendance, day

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
## Adapter
2+
[Convert State Data] => Dimension/Dataset events [Ramya]
3+
Gradeee 1 => 2
4+
5+
## Data from the state/NAS/VSK
6+
grade, attendance, day
7+
Grade 1, 20, 2023-2-2
8+
Classsses 1, 20, 2023-2-2
9+
Grade 2, 20, 2023-2-2
10+
11+
## Grade Dimension
12+
grade_id, grade_number, grade_nas, grade_vsk, grade_name_hp
13+
1, 1, Grade 1, Class 1, Grade 1
14+
2, 1, Gradeee 1, Classes 1, Gradeee 1
15+
16+
## Adapter
17+
Step 1: Pick the ID from the dimension table called Grade.
18+
Step 2: Map it back to data and get the ID.

‎docs/query-builder/prisma-types.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
A refactor should happen at this level
2+
3+
- https://github.com/amplication/prisma-schema-dsl
4+
- Figure out how to manage changes in schema though Git - see amplication

‎impl/c-qube/ingest/config.json

Lines changed: 0 additions & 135 deletions
Original file line numberDiff line numberDiff line change
@@ -34,141 +34,6 @@
3434
],
3535
"blacklisted": []
3636
}
37-
},
38-
{
39-
"name": "PM Poshan",
40-
"namespace": "pm_poshan",
41-
"description": "PM Poshan",
42-
"shouldIngestToDB": true,
43-
"input": {
44-
"files": "./ingest/programs/pm-poshan"
45-
},
46-
"./output": {
47-
"location": "./output/programs/pm-poshan"
48-
},
49-
"dimensions": {
50-
"whitelisted": [
51-
"district,categorypm"
52-
],
53-
"blacklisted": []
54-
}
55-
},
56-
{
57-
"name": "NAS",
58-
"namespace": "nas",
59-
"description": "NAS",
60-
"shouldIngestToDB": true,
61-
"input": {
62-
"files": "./ingest/programs/nas"
63-
},
64-
"./output": {
65-
"location": "./output/programs/nas"
66-
},
67-
"dimensions": {
68-
"whitelisted": [
69-
"district,lo,subject,grade",
70-
"state,lo,subject,grade"
71-
],
72-
"blacklisted": []
73-
}
74-
},
75-
{
76-
"name": "UDISE",
77-
"namespace": "udise",
78-
"description": "UDISE",
79-
"shouldIngestToDB": true,
80-
"input": {
81-
"files": "./ingest/programs/udise"
82-
},
83-
"./output": {
84-
"location": "./output/programs/udise"
85-
},
86-
"dimensions": {
87-
"whitelisted": [
88-
"district,categoryudise",
89-
"state,categoryudise"
90-
],
91-
"blacklisted": []
92-
}
93-
},
94-
{
95-
"name": "Review and Monitoring",
96-
"namespace": "rev_and_monitoring",
97-
"description": "Review and Monitoring",
98-
"shouldIngestToDB": true,
99-
"input": {
100-
"files": "./ingest/programs/rev-and-monitor"
101-
},
102-
"./output": {
103-
"location": "./output/programs/rev-and-monitor"
104-
},
105-
"dimensions": {
106-
"whitelisted": [
107-
"academicyear,district",
108-
"academicyear,block",
109-
"academicyear,cluster"
110-
],
111-
"blacklisted": []
112-
}
113-
},
114-
{
115-
"name": "DIKSHA",
116-
"namespace": "diksha",
117-
"description": "DIKSHA",
118-
"shouldIngestToDB": true,
119-
"input": {
120-
"files": "./ingest/programs/diksha"
121-
},
122-
"./output": {
123-
"location": "./output/programs/diksha"
124-
},
125-
"dimensions": {
126-
"whitelisted": [
127-
"state,grade,subject,medium,board",
128-
"textbookdiksha,grade,subject,medium",
129-
"textbookdiksha,grade,subject,medium"
130-
],
131-
"blacklisted": []
132-
}
133-
},
134-
{
135-
"name": "PGI",
136-
"namespace": "pgi",
137-
"description": "PGI",
138-
"shouldIngestToDB": true,
139-
"input": {
140-
"files": "./ingest/programs/pgi"
141-
},
142-
"./output": {
143-
"location": "./output/programs/pgi"
144-
},
145-
"dimensions": {
146-
"whitelisted": [
147-
"state,district,categorypgi",
148-
"state,categorypgi"
149-
],
150-
"blacklisted": []
151-
}
152-
},
153-
{
154-
"name": "NISHTHA",
155-
"namespace": "nishtha",
156-
"description": "NISHTHA",
157-
"shouldIngestToDB": true,
158-
"input": {
159-
"files": "./ingest/programs/nishtha"
160-
},
161-
"./output": {
162-
"location": "./output/programs/nishtha"
163-
},
164-
"dimensions": {
165-
"whitelisted": [
166-
"state,district,programnishtha",
167-
"state,programnishtha,coursenishtha",
168-
"state,programnishtha"
169-
],
170-
"blacklisted": []
171-
}
17237
}
17338
]
17439
}

‎impl/c-qube/ingest/config.old.json

Lines changed: 83 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
{
2+
"globals": {
3+
"onlyCreateWhitelisted": true
4+
},
25
"dimensions": {
36
"namespace": "dimensions",
47
"fileNameFormat": "${dimensionName}.${index}.dimensions.data.csv",
@@ -7,29 +10,9 @@
710
}
811
},
912
"programs": [
10-
{
11-
"name": "Review and Monitoring",
12-
"namespace": "rev_and_monitoring",
13-
"description": "Review and Monitoring",
14-
"shouldIngestToDB": true,
15-
"input": {
16-
"files": "./ingest/programs/rev-and-monitor"
17-
},
18-
"./output": {
19-
"location": "./output/programs/rev-and-monitor"
20-
},
21-
"dimensions": {
22-
"whitelisted": [
23-
"academicyear,district",
24-
"academicyear,block",
25-
"academicyear,cluster"
26-
],
27-
"blacklisted": []
28-
}
29-
},
3013
{
3114
"name": "School Attendance",
32-
"namespace": "school_attendance",
15+
"namespace": "sch_att",
3316
"description": "School Attendance",
3417
"shouldIngestToDB": true,
3518
"input": {
@@ -53,20 +36,19 @@
5336
}
5437
},
5538
{
56-
"name": "UDISE",
57-
"namespace": "udise",
58-
"description": "UDISE",
39+
"name": "PM Poshan",
40+
"namespace": "pm_poshan",
41+
"description": "PM Poshan",
5942
"shouldIngestToDB": true,
6043
"input": {
61-
"files": "./ingest/programs/udise"
44+
"files": "./ingest/programs/pm-poshan"
6245
},
6346
"./output": {
64-
"location": "./output/programs/udise"
47+
"location": "./output/programs/pm-poshan"
6548
},
6649
"dimensions": {
6750
"whitelisted": [
68-
"district,udisecategory",
69-
"state,udisecategory"
51+
"district,categorypm"
7052
],
7153
"blacklisted": []
7254
}
@@ -84,8 +66,67 @@
8466
},
8567
"dimensions": {
8668
"whitelisted": [
87-
"district,nasindicator,nassubject,grade",
88-
"state,nasindicator,nassubject,grade"
69+
"district,lo,subject,grade",
70+
"state,lo,subject,grade"
71+
],
72+
"blacklisted": []
73+
}
74+
},
75+
{
76+
"name": "UDISE",
77+
"namespace": "udise",
78+
"description": "UDISE",
79+
"shouldIngestToDB": true,
80+
"input": {
81+
"files": "./ingest/programs/udise"
82+
},
83+
"./output": {
84+
"location": "./output/programs/udise"
85+
},
86+
"dimensions": {
87+
"whitelisted": [
88+
"district,categoryudise",
89+
"state,categoryudise"
90+
],
91+
"blacklisted": []
92+
}
93+
},
94+
{
95+
"name": "Review and Monitoring",
96+
"namespace": "rev_and_monitoring",
97+
"description": "Review and Monitoring",
98+
"shouldIngestToDB": true,
99+
"input": {
100+
"files": "./ingest/programs/rev-and-monitor"
101+
},
102+
"./output": {
103+
"location": "./output/programs/rev-and-monitor"
104+
},
105+
"dimensions": {
106+
"whitelisted": [
107+
"academicyear,district",
108+
"academicyear,block",
109+
"academicyear,cluster"
110+
],
111+
"blacklisted": []
112+
}
113+
},
114+
{
115+
"name": "DIKSHA",
116+
"namespace": "diksha",
117+
"description": "DIKSHA",
118+
"shouldIngestToDB": true,
119+
"input": {
120+
"files": "./ingest/programs/diksha"
121+
},
122+
"./output": {
123+
"location": "./output/programs/diksha"
124+
},
125+
"dimensions": {
126+
"whitelisted": [
127+
"state,grade,subject,medium,board",
128+
"textbookdiksha,grade,subject,medium",
129+
"textbookdiksha,grade,subject,medium"
89130
],
90131
"blacklisted": []
91132
}
@@ -103,25 +144,29 @@
103144
},
104145
"dimensions": {
105146
"whitelisted": [
106-
"district,category",
107-
"state,category"
147+
"state,district,categorypgi",
148+
"state,categorypgi"
108149
],
109150
"blacklisted": []
110151
}
111152
},
112153
{
113-
"name": "PM-Poshan",
114-
"namespace": "pm-poshan",
115-
"description": "PM-Poshan",
154+
"name": "NISHTHA",
155+
"namespace": "nishtha",
156+
"description": "NISHTHA",
116157
"shouldIngestToDB": true,
117158
"input": {
118-
"files": "./ingest/programs/pm-poshan"
159+
"files": "./ingest/programs/nishtha"
119160
},
120161
"./output": {
121-
"location": "./output/programs/pgi"
162+
"location": "./output/programs/nishtha"
122163
},
123164
"dimensions": {
124-
"whitelisted": [],
165+
"whitelisted": [
166+
"state,district,programnishtha",
167+
"state,programnishtha,coursenishtha",
168+
"state,programnishtha"
169+
],
125170
"blacklisted": []
126171
}
127172
}

‎impl/c-qube/ingest/dimensions/grade-dimension.grammar.csv

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
PK,Index,,
1+
PK,Index,Index,Index
22
string,string,string,string
3-
grade_id,grade_nas,grade_diksha,grade_state
3+
grade_id,grade_nas,grade_diksha,grade_state

0 commit comments

Comments
 (0)