Skip to content

Commit 62242d3

Browse files
janeenyamak1janeenyamak1
authored
TDS Write: generate processing date (finos#3951)
1 parent 0cf893d commit 62242d3

File tree

19 files changed

+1498
-81
lines changed

19 files changed

+1498
-81
lines changed

‎legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/test/java/org/finos/legend/engine/server/TestSemiStructuredWrite.java‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ protected void insertTestData(int serverPort) throws SQLException
154154
protected void insertTestData(Statement statement) throws SQLException
155155
{
156156
statement.execute("Drop table if exists Person_Table;");
157-
statement.execute("Create Table Person_Table(FIRM VARCHAR(100) , FIRST_NAME VARCHAR(100) , KERBEROS VARCHAR(100) , LAST_NAME VARCHAR(100))");
157+
statement.execute("Create Table Person_Table(FIRM VARCHAR(100) , FIRST_NAME VARCHAR(100) , KERBEROS VARCHAR(100) , LAST_NAME VARCHAR(100), IN_Z TIMESTAMP, OUT_Z TIMESTAMP)");
158158
}
159159

160160
@Test
@@ -178,6 +178,26 @@ public void testSemiStructuredWrite() throws Exception
178178

179179
}
180180

181+
@Test
182+
public void testSemiStructuredWriteWithProcessingDate() throws Exception
183+
{
184+
String input = "{\n" +
185+
" \"kerberos\": \"kerberos 98\",\n" +
186+
" \"firstName\": \"firstName 98\",\n" +
187+
" \"lastName\": \"lastName 56\",\n" +
188+
" \"firm\": {\n" +
189+
" \"name\": \"name 71\",\n" +
190+
" \"numberOfEmployees\": 54\n" +
191+
" }\n" +
192+
"}";
193+
String plan = readContent(modelResourcePathSemiStructuredWithProcessingDate());
194+
SingleExecutionPlan executionPlan = objectMapper.readValue(plan, SingleExecutionPlan.class);
195+
Map inputMap = Maps.mutable.empty();
196+
inputMap.put("jsonString",input);
197+
Result result = planExecutor.execute(executionPlan, inputMap);
198+
Assert.assertTrue("Success - 1 rows updated!".equals(((ConstantResult)result).getValue()), () -> String.format("Write Failed. Result: %s", ((ConstantResult)result).getValue()));
199+
}
200+
181201
public void testStatement(String s)
182202
{
183203
testRelationalExecutor = buildTestExecutor(serverPort);
@@ -202,6 +222,11 @@ public String modelResourcePathSemiStructured()
202222
return "/org/finos/legend/engine/server/testSemiStructured.json";
203223
}
204224

225+
public String modelResourcePathSemiStructuredWithProcessingDate()
226+
{
227+
return "/org/finos/legend/engine/server/testSemiStructuredWithProcessingDate.json";
228+
}
229+
205230
private String readContent(String resourcePath)
206231
{
207232
try (BufferedReader buffer = new BufferedReader(new InputStreamReader(Objects.requireNonNull(TestSemiStructuredWrite.class.getResourceAsStream(resourcePath)))))

‎legend-engine-config/legend-engine-server/legend-engine-server-http-server/src/test/resources/org/finos/legend/engine/server/testSemiStructuredWithProcessingDate.json‎

Lines changed: 384 additions & 0 deletions
Large diffs are not rendered by default.

‎legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/executionPlan/platformBinding/typeInfo/typeInfo.pure‎

Lines changed: 36 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ function meta::pure::executionPlan::platformBinding::typeInfo::addForClass(info:
111111
{p, i|
112112
$i->addBasicTypeInfoIfMissing($p.genericType.rawType->toOne())
113113
->addPropertyForClassIfMissing($class, $p);
114-
},
114+
},
115115
$basic
116116
),
117117
| $basic
@@ -128,7 +128,7 @@ function meta::pure::executionPlan::platformBinding::typeInfo::addForClass(info:
128128
| $class.propertiesFromAssociations->fold({p,i | $i->addPropertyForClassIfMissing($class, $p)}, $plusDerived),
129129
| $plusDerived
130130
);
131-
131+
132132
$withAssoc->forClass($class).supertypes->fold({c, set | $set->addForClass($c, $includeSimple, $includeComplex, $includeDerived, $includeAssociations)}, $withAssoc);
133133
}
134134

@@ -213,11 +213,16 @@ function meta::pure::executionPlan::platformBinding::typeInfo::allProperties(inf
213213
$info->allProperties($info->forClass($class))
214214
}
215215

216+
function meta::pure::executionPlan::platformBinding::typeInfo::allPropertiesWithoutGeneratedMilestoning(info:TypeInfoSet[1], class:Class<Any>[1]): Property<Nil,Any|*>[*]
217+
{
218+
$info->allProperties($info->forClass($class))->filter(p | !$p->hasGeneratedMilestoningPropertyStereotype())
219+
}
220+
216221
function meta::pure::executionPlan::platformBinding::typeInfo::allProperties(info:TypeInfoSet[1], classInfo:ClassTypeInfo[1]): Property<Nil,Any|*>[*]
217222
{
218223
$classInfo.supertypes->fold(
219224
{supertype, properties|
220-
let notOverriden = $info->forClass($supertype).properties->filter(p| !$p.name->in($properties.name));
225+
let notOverriden = $info->forClass($supertype).properties->filter(p| !$p.name->in($properties.name));
221226
$properties->concatenate($notOverriden);
222227
},
223228
$classInfo.properties
@@ -233,7 +238,7 @@ function meta::pure::executionPlan::platformBinding::typeInfo::allQualifiedPrope
233238
{
234239
$classInfo.supertypes->fold(
235240
{supertype, qualifiedProperties|
236-
let notOverriden = $info->forClass($supertype).qualifiedProperties->filter(p| !$p.name->in($qualifiedProperties.name));
241+
let notOverriden = $info->forClass($supertype).qualifiedProperties->filter(p| !$p.name->in($qualifiedProperties.name));
237242
$qualifiedProperties->concatenate($notOverriden);
238243
},
239244
$classInfo.qualifiedProperties
@@ -279,7 +284,7 @@ function meta::pure::executionPlan::platformBinding::typeInfo::enumDependenciesV
279284
{
280285
let directProperties = $info->allProperties($class);
281286
let dependencyProperties = $info->classDependenciesViaPropertiesWithSubTypes($class)->map(c| $info->allProperties($c));
282-
287+
283288
$directProperties
284289
->concatenate($dependencyProperties)
285290
->filter(p| $p.genericType.rawType->toOne()->instanceOf(Enumeration))
@@ -295,7 +300,7 @@ function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo
295300
{|
296301
let forThisClass = $info->allProperties($class)->filter(p| $p.genericType.rawType->toOne()->instanceOf(Class));
297302
let forThisWithSubtypes = $forThisClass->concatenate($class->getSpecializations()->filter(s | $info.typeInfos->exists(ti | $ti.type == $s->cast(@Type)))->map(g | $info->allProperties($g)->filter(p| $p.genericType.rawType->toOne()->instanceOf(Class))));
298-
303+
299304
let forChildren = $forThisWithSubtypes.genericType.rawType->cast(@Class<Any>)->removeDuplicatesBy(classEqualityKey())->map(c| $info->classPropertiesRecursiveWithsubTypes($c, $visitedClasses->concatenate($class)));
300305
$forThisWithSubtypes->concatenate($forChildren);
301306
}
@@ -316,12 +321,12 @@ function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo
316321
),
317322
st: SubTypeGraphFetchTree[1] | $st.subTypeClass,
318323
r: RootGraphFetchTree<Any>[1] | $r.class,
319-
p: PropertyGraphFetchTree[1] | $p.property.genericType.rawType->concatenate($p.subType)
324+
p: PropertyGraphFetchTree[1] | $p.property.genericType.rawType->concatenate($p.subType)
320325
]);
321326
let withThisType = $types->fold({t, tis| $tis->addBasicTypeInfoIfMissing($t)}, $info);
322-
327+
323328
let classes = $types->filter(t| $t->instanceOf(Class))->cast(@Class<Any>);
324-
329+
325330
let milestoneProperties = $classes.properties->filter(p|$p->isPrimitiveValueProperty() && $p->hasGeneratedMilestoningPropertyStereotype());
326331
let withMilestoning = $withThisType->addProperties($milestoneProperties);
327332

@@ -333,26 +338,26 @@ function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo
333338
gft : GraphFetchTree[1] | []
334339
]);
335340
let withRequiredQualifiers = $requiredQP->fold({qp,tis |$tis->addQualifiedPropertyForTypeIfMissing($qp)}, $withMilestoning);
336-
341+
337342
let withPropertySubtrees= $tree.subTrees->map(x | $x->meta::pure::graphFetch::routing::byPassClusteringInfo())->cast(@PropertyGraphFetchTree)->fold(
338343
{pgft, tis|
339-
if($pgft.property->instanceOf(QualifiedProperty),
340-
| $tis->addQualifiedPropertyForTypeIfMissing($pgft.property->cast(@QualifiedProperty<Any>)),
344+
if($pgft.property->instanceOf(QualifiedProperty),
345+
| $tis->addQualifiedPropertyForTypeIfMissing($pgft.property->cast(@QualifiedProperty<Any>)),
341346
| $classes->fold({c, tis2| $tis2->addPropertyForClassIfDefined($c, $pgft.property.name->toOne(), false)}, $tis)
342347
)->typeInfoFromTree($pgft);
343-
},
348+
},
344349
$withRequiredQualifiers
345350
);
346351

347352
let withSubTypeTrees=if($tree->instanceOf(RootGraphFetchTree),
348353
| $tree.subTypeTrees->map(x | $x->meta::pure::graphFetch::routing::byPassClusteringInfo())->cast(@SubTypeGraphFetchTree)->fold(
349354
{strgft, tis|
350355
$tis->typeInfoFromTree($strgft);
351-
},
356+
},
352357
$withPropertySubtrees
353358
);,
354359
| $withPropertySubtrees);
355-
$withSubTypeTrees;
360+
$withSubTypeTrees;
356361
}
357362

358363
function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo::constraintInfoFromTree(typeInfos:TypeInfoSet[1], tree:GraphFetchTree[1]): TypeInfoSet[1]
@@ -423,7 +428,7 @@ function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo
423428
| $set.class
424429
);
425430
$updatedTypes->addForClass($type, true, false, false, false);
426-
},
431+
},
427432
$info
428433
);
429434
}
@@ -435,7 +440,7 @@ function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo
435440

436441
function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo::addPropertyForTypeIfMissing(info:TypeInfoSet[1], property:Property<Nil,Any|*>[1], isNewInstanceAllowed:Boolean[1]): TypeInfoSet[1]
437442
{
438-
443+
439444
$property.owner->match([
440445
{c: Class<Any>[1] |
441446
$info->addPropertyForClassIfMissing($c, $property, $isNewInstanceAllowed);
@@ -445,7 +450,7 @@ function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo
445450
let c1 = $a.properties->at(1).genericType.rawType->cast(@Class<Any>)->toOne();
446451
let p2 = $a.properties->at(1);
447452
let c2 = $a.properties->at(0).genericType.rawType->cast(@Class<Any>)->toOne();
448-
453+
449454
$info->addPropertyForClassIfMissing($c1, $p1)->addPropertyForClassIfMissing($c2, $p2, $isNewInstanceAllowed);
450455
}
451456
]);
@@ -454,9 +459,9 @@ function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo
454459
function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo::addPropertyForClassIfDefined(info:TypeInfoSet[1], class:Class<Any>[1], propertyName:String[1], isNewInstanceAllowed:Boolean[1]): TypeInfoSet[1]
455460
{
456461
let property = $class.properties->concatenate($class.propertiesFromAssociations)->filter(p| $p.name == $propertyName);
457-
462+
458463
$property->match([
459-
{p:Property<Nil,Any|*>[1] |
464+
{p:Property<Nil,Any|*>[1] |
460465
$p.owner->match([
461466
{c: Class<Any>[1] |
462467
$info->addPropertyForClassIfMissing($c, $p, $isNewInstanceAllowed);
@@ -466,12 +471,12 @@ function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo
466471
let c1 = $a.properties->at(1).genericType.rawType->cast(@Class<Any>)->toOne();
467472
let p2 = $a.properties->at(1);
468473
let c2 = $a.properties->at(0).genericType.rawType->cast(@Class<Any>)->toOne();
469-
474+
470475
$info->addPropertyForClassIfMissing($c1, $p1)->addPropertyForClassIfMissing($c2, $p2, $isNewInstanceAllowed);
471476
}
472477
]);
473478
},
474-
{none:Property<Nil,Any|*>[0] |
479+
{none:Property<Nil,Any|*>[0] |
475480
$class->getGeneralizations()->fold({c, tis| $tis->addPropertyForClassIfDefined($c, $propertyName, $isNewInstanceAllowed)}, $info);
476481
},
477482
{multiple:Property<Nil,Any|*>[*] |
@@ -483,18 +488,18 @@ function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo
483488

484489
function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo::addPropertyForClassIfMissing(info:TypeInfoSet[1], class:Class<Any>[1], property:Property<Nil,Any|*>[1]): TypeInfoSet[1]
485490
{
486-
addPropertyForClassIfMissing($info, $class, $property, false);
491+
addPropertyForClassIfMissing($info, $class, $property, false);
487492
}
488493

489494
function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo::addPropertyForClassIfMissing(info:TypeInfoSet[1], class:Class<Any>[1], property:Property<Nil,Any|*>[1], isNewInstanceAllowed:Boolean[1]): TypeInfoSet[1]
490-
{
495+
{
491496
let enriched = $info->addBasicTypeInfoIfMissing($class, $isNewInstanceAllowed);
492497
let ti = $enriched->forClass($class);
493498
let rtnType = $property->functionReturnType().rawType->toOne();
494499
let enriched2 = if($rtnType->instanceOf(Class) || $rtnType->instanceOf(Enumeration), | $enriched->addBasicTypeInfoIfMissing($rtnType, $isNewInstanceAllowed), | $enriched);
495500

496-
let withProperty = if($ti.properties->exists(p| $p.name == $property.name),
497-
| $enriched2,
501+
let withProperty = if($ti.properties->exists(p| $p.name == $property.name),
502+
| $enriched2,
498503
|$enriched2->addOrReplace(^$ti(properties=$ti.properties->concatenate($property)->sortProperties($class)))
499504
);
500505

@@ -573,19 +578,19 @@ function<<access.private>> meta::pure::executionPlan::platformBinding::typeInfo:
573578

574579
function<<access.private>> meta::pure::executionPlan::platformBinding::typeInfo::addBasicTypeInfoIfMissing(info:TypeInfoSet[1], type:Type[1], isNewInstanceAllowed:Boolean[1]):TypeInfoSet[1]
575580
{
576-
581+
577582
addBasicTypeInfoIfMissing($info,[],$type, $isNewInstanceAllowed);
578583
}
579584

580585
function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo::addBasicTypeInfoIfMissing(info:TypeInfoSet[1], seenClasses:Class<Any>[*], type:Type[1]): TypeInfoSet[1]
581586
{
582587
addBasicTypeInfoIfMissing($info, $seenClasses, $type, false);
583588
}
584-
589+
585590
function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo::addBasicTypeInfoIfMissing(info:TypeInfoSet[1], seenClasses:Class<Any>[*], type:Type[1], isNewInstanceAllowed:Boolean[1]): TypeInfoSet[1]
586591
{
587592
if($info.typeInfos.type->contains($type),
588-
{|
593+
{|
589594
$type->match([
590595
c: Class<Any>[1] | let newTypeInfos = $info.typeInfos->fold({typeInfo, typeInfoList | if($typeInfo.type == $type,
591596
| let cti = $typeInfo->cast(@ClassTypeInfo);
@@ -597,7 +602,7 @@ function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo
597602

598603
])
599604
},
600-
{|
605+
{|
601606
$type->match([
602607
e : Enumeration<Any>[1] | $info->addEnumerationTypeInfo($e),
603608
c : Class<Any>[1] | $info->addBasicClassTypeInfo($c, $seenClasses, $isNewInstanceAllowed),
@@ -672,7 +677,7 @@ function <<access.private>> meta::pure::executionPlan::platformBinding::typeInf
672677
function <<access.private>> meta::pure::executionPlan::platformBinding::typeInfo::getSpecializations(class:Type[1]):Class<Any>[*]
673678
{
674679
let specializations = $class.specializations->map(g | $g.specific->filter(t|$t->instanceOf(Class))->cast(@Class<Any>));
675-
680+
676681
$specializations
677682
->concatenate($specializations->map(g| $g->getSpecializations()))
678683
->filter(meta::pure::metamodel::type::Any->isNotClassFn())

0 commit comments

Comments
 (0)