Skip to content

Commit b338b0c

Browse files
YoguYogu
committed
perf: hoist root field calculations out of loops to reduce query memory usage
ArangoDB also has logic to hoist variable assignments if it can, but it sometimes (or rather often in our cases) pushes them down again. This is problematic if the result of the assignment is much smaller than its dependencies. In the case of root entities that are accessed via @root, this is often the case because usually, only a subset of the root's fields are accessed within the loop, but a lot more fields are accessed on root level. We can prevent ArangoDB from pushing down the variables again by wrapping their assignment in NOEVAL(...) If a query requests many fields from the root object in general (by default 5), the reduce-extraction-to-projection optimization is no longer applied so the full root object is held in memory, and without variable hosting, can be duplicated for each result item of a traversal, leading to very high memory usage (root entity size * number of collect items) In the regression test root-fields/root-with-collect in query q, the memory usage increased slightly. This is likely because the added variable assignments have an overhead that is not offset by the hoisting because of the small number of items and small size of objects in the regression tests.
1 parent b51379e commit b338b0c

File tree

22 files changed

+743
-287
lines changed

22 files changed

+743
-287
lines changed

‎spec/regression/logistics/tests/reference-to-id/aql/referenceToExistingID.aql‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,21 +11,19 @@ RETURN {
1111
"items": (
1212
FOR v_deliveryItem1
1313
IN v_delivery1.`items`[*]
14+
LET v_handlingUnit1 = (IS_NULL(v_deliveryItem1.`handlingUnit`) ? null : FIRST((
15+
FOR v_handlingUnit2
16+
IN @@handlingUnits
17+
FILTER ((v_handlingUnit2._key > NULL) && (v_handlingUnit2._key == v_deliveryItem1.`handlingUnit`))
18+
LIMIT @var3
19+
RETURN v_handlingUnit2
20+
)))
1421
RETURN {
1522
"itemNumber": v_deliveryItem1.`itemNumber`,
16-
"handlingUnit": FIRST(
17-
LET v_handlingUnit1 = (IS_NULL(v_deliveryItem1.`handlingUnit`) ? null : FIRST((
18-
FOR v_handlingUnit2
19-
IN @@handlingUnits
20-
FILTER ((v_handlingUnit2._key > NULL) && (v_handlingUnit2._key == v_deliveryItem1.`handlingUnit`))
21-
LIMIT @var3
22-
RETURN v_handlingUnit2
23-
)))
24-
RETURN (IS_NULL(v_handlingUnit1) ? null : {
25-
"id": v_handlingUnit1._key,
26-
"huNumber": v_handlingUnit1.`huNumber`
27-
})
28-
)
23+
"handlingUnit": (IS_NULL(v_handlingUnit1) ? null : {
24+
"id": v_handlingUnit1._key,
25+
"huNumber": v_handlingUnit1.`huNumber`
26+
})
2927
}
3028
)
3129
})

‎spec/regression/logistics/tests/reference-to-id/aql/referenceToWrongID.aql‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,24 +11,22 @@ RETURN {
1111
"items": (
1212
FOR v_deliveryItem1
1313
IN v_delivery1.`items`[*]
14+
LET v_handlingUnit1 = (IS_NULL(v_deliveryItem1.`handlingUnit`) ? null : FIRST((
15+
FOR v_handlingUnit2
16+
IN @@handlingUnits
17+
FILTER ((v_handlingUnit2._key > NULL) && (v_handlingUnit2._key == v_deliveryItem1.`handlingUnit`))
18+
LIMIT @var3
19+
RETURN v_handlingUnit2
20+
)))
1421
RETURN {
1522
"itemNumber": v_deliveryItem1.`itemNumber`,
16-
"handlingUnit": FIRST(
17-
LET v_handlingUnit1 = (IS_NULL(v_deliveryItem1.`handlingUnit`) ? null : FIRST((
18-
FOR v_handlingUnit2
19-
IN @@handlingUnits
20-
FILTER ((v_handlingUnit2._key > NULL) && (v_handlingUnit2._key == v_deliveryItem1.`handlingUnit`))
21-
LIMIT @var3
22-
RETURN v_handlingUnit2
23-
)))
24-
RETURN (IS_NULL(v_handlingUnit1) ? null : {
25-
"id": v_handlingUnit1._key,
26-
"huNumber": v_handlingUnit1.`huNumber`
27-
})
28-
)
23+
"handlingUnit": (IS_NULL(v_handlingUnit1) ? null : {
24+
"id": v_handlingUnit1._key,
25+
"huNumber": v_handlingUnit1.`huNumber`
26+
})
2927
}
3028
)
3129
})
3230
}
3331

34-
// Peak memory usage: 98304 bytes
32+
// Peak memory usage: 65536 bytes

‎spec/regression/papers/tests/references/aql/references.aql‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,20 @@ RETURN {
88
"literatureReferences": (
99
FOR v_literatureReference1
1010
IN v_paper1.`literatureReferences`[*]
11+
LET v_paper2 = (IS_NULL(v_literatureReference1.`paper`) ? null : FIRST((
12+
FOR v_paper3
13+
IN @@papers
14+
FILTER ((v_paper3.`key` > NULL) && (v_paper3.`key` == v_literatureReference1.`paper`))
15+
LIMIT @var1
16+
RETURN v_paper3
17+
)))
1118
RETURN {
12-
"paper": FIRST(
13-
LET v_paper2 = (IS_NULL(v_literatureReference1.`paper`) ? null : FIRST((
14-
FOR v_paper3
15-
IN @@papers
16-
FILTER ((v_paper3.`key` > NULL) && (v_paper3.`key` == v_literatureReference1.`paper`))
17-
LIMIT @var1
18-
RETURN v_paper3
19-
)))
20-
RETURN (IS_NULL(v_paper2) ? null : {
21-
"title": v_paper2.`title`,
22-
"publishDate": v_paper2.`publishDate`,
23-
"isPublished": v_paper2.`isPublished`,
24-
"tags": v_paper2.`tags`[*]
25-
})
26-
),
19+
"paper": (IS_NULL(v_paper2) ? null : {
20+
"title": v_paper2.`title`,
21+
"publishDate": v_paper2.`publishDate`,
22+
"isPublished": v_paper2.`isPublished`,
23+
"tags": v_paper2.`tags`[*]
24+
}),
2725
"authors": v_literatureReference1.`authors`[*],
2826
"pages": (IS_NULL(v_literatureReference1.`pages`) ? null : {
2927
"startPage": v_literatureReference1.`pages`.`startPage`,
@@ -36,4 +34,4 @@ RETURN {
3634
)
3735
}
3836

39-
// Peak memory usage: 131072 bytes
37+
// Peak memory usage: 98304 bytes

‎spec/regression/root-fields/tests/root-and-parent-with-collect/aql/q.aql‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,14 @@ RETURN {
2828
"rootGrandchildren": (
2929
FOR v_root2, v_edge1, v_path1 IN @var5..@var6 OUTBOUND v_root21 @@root2s_roots
3030
FILTER v_root2._key != null
31+
LET v_root3 = NOEVAL({
32+
"name": v_root2.`name`
33+
})
3134
FOR v_item1 IN v_root2.`children`[*].`children`[*][** RETURN {
3235
value: {
3336
"name": CURRENT.`name`,
3437
"parent": { __cruddl_runtime_error_code: @var7, __cruddl_runtime_error: @var8 },
35-
"root": {
36-
"name": v_root2.`name`
37-
}
38+
"root": v_root3
3839
},
3940
sortValues: [
4041
CURRENT.`name`
@@ -44,15 +45,16 @@ RETURN {
4445
RETURN v_item1.value
4546
),
4647
"rootExtensionGrandchildren": (
47-
FOR v_root3, v_edge2, v_path2 IN @var9..@var10 OUTBOUND v_root21 @@root2s_roots
48-
FILTER v_root3._key != null
49-
FOR v_item2 IN v_root3.`children`[*].`extension`.`children`[*][** RETURN {
48+
FOR v_root4, v_edge2, v_path2 IN @var9..@var10 OUTBOUND v_root21 @@root2s_roots
49+
FILTER v_root4._key != null
50+
LET v_root5 = NOEVAL({
51+
"name": v_root4.`name`
52+
})
53+
FOR v_item2 IN v_root4.`children`[*].`extension`.`children`[*][** RETURN {
5054
value: {
5155
"name": CURRENT.`name`,
5256
"parent": { __cruddl_runtime_error_code: @var11, __cruddl_runtime_error: @var12 },
53-
"root": {
54-
"name": v_root3.`name`
55-
}
57+
"root": v_root5
5658
},
5759
sortValues: [
5860
CURRENT.`name`

‎spec/regression/root-fields/tests/root-and-parent/aql/test.aql‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ RETURN {
66
RETURN {
77
"name": v_root1.`name`,
88
"children": (
9+
LET v_root2 = NOEVAL({
10+
"name": v_root1.`name`
11+
})
912
FOR v_child1
1013
IN v_root1.`children`[*]
1114
RETURN {
@@ -34,9 +37,7 @@ RETURN {
3437
"parent": {
3538
"name": v_root1.`name`
3639
},
37-
"root": {
38-
"name": v_root1.`name`
39-
}
40+
"root": v_root2
4041
}
4142
)
4243
}

‎spec/regression/root-fields/tests/root-with-collect/aql/filter.aql‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,13 @@ RETURN {
4343
"rootGrandchildren": (
4444
FOR v_root2, v_edge1, v_path1 IN @var11..@var12 OUTBOUND v_root21 @@root2s_roots
4545
FILTER v_root2._key != null
46+
LET v_root3 = NOEVAL({
47+
"name": v_root2.`name`
48+
})
4649
FOR v_item3 IN v_root2.`children`[*].`children`[*][** FILTER (RIGHT(CURRENT.`name`, LENGTH(@var13)) == @var14) RETURN {
4750
value: {
4851
"name": CURRENT.`name`,
49-
"root": {
50-
"name": v_root2.`name`
51-
}
52+
"root": v_root3
5253
},
5354
sortValues: [
5455
CURRENT.`name`
@@ -61,24 +62,25 @@ RETURN {
6162
"count": FIRST(
6263
FOR v_item4
6364
IN (
64-
FOR v_root3, v_edge2, v_path2 IN @var16..@var17 OUTBOUND v_root21 @@root2s_roots
65-
FILTER v_root3._key != null
66-
FOR v_item5 IN v_root3.`children`[*].`children`[*][** FILTER (RIGHT(CURRENT.`name`, LENGTH(@var18)) == @var19)]
65+
FOR v_root4, v_edge2, v_path2 IN @var16..@var17 OUTBOUND v_root21 @@root2s_roots
66+
FILTER v_root4._key != null
67+
FOR v_item5 IN v_root4.`children`[*].`children`[*][** FILTER (RIGHT(CURRENT.`name`, LENGTH(@var18)) == @var19)]
6768
RETURN v_item5
6869
)
6970
COLLECT WITH COUNT INTO v_count3
7071
RETURN v_count3
7172
)
7273
},
7374
"rootExtensionGrandchildren": (
74-
FOR v_root4, v_edge3, v_path3 IN @var20..@var21 OUTBOUND v_root21 @@root2s_roots
75-
FILTER v_root4._key != null
76-
FOR v_item6 IN v_root4.`children`[*].`extension`.`children`[*][** FILTER (RIGHT(CURRENT.`name`, LENGTH(@var22)) == @var23) RETURN {
75+
FOR v_root5, v_edge3, v_path3 IN @var20..@var21 OUTBOUND v_root21 @@root2s_roots
76+
FILTER v_root5._key != null
77+
LET v_root6 = NOEVAL({
78+
"name": v_root5.`name`
79+
})
80+
FOR v_item6 IN v_root5.`children`[*].`extension`.`children`[*][** FILTER (RIGHT(CURRENT.`name`, LENGTH(@var22)) == @var23) RETURN {
7781
value: {
7882
"name": CURRENT.`name`,
79-
"root": {
80-
"name": v_root4.`name`
81-
}
83+
"root": v_root6
8284
},
8385
sortValues: [
8486
CURRENT.`name`
@@ -91,9 +93,9 @@ RETURN {
9193
"count": FIRST(
9294
FOR v_item7
9395
IN (
94-
FOR v_root5, v_edge4, v_path4 IN @var25..@var26 OUTBOUND v_root21 @@root2s_roots
95-
FILTER v_root5._key != null
96-
FOR v_item8 IN v_root5.`children`[*].`extension`.`children`[*][** FILTER (RIGHT(CURRENT.`name`, LENGTH(@var27)) == @var28)]
96+
FOR v_root7, v_edge4, v_path4 IN @var25..@var26 OUTBOUND v_root21 @@root2s_roots
97+
FILTER v_root7._key != null
98+
FOR v_item8 IN v_root7.`children`[*].`extension`.`children`[*][** FILTER (RIGHT(CURRENT.`name`, LENGTH(@var27)) == @var28)]
9799
RETURN v_item8
98100
)
99101
COLLECT WITH COUNT INTO v_count4

‎spec/regression/root-fields/tests/root-with-collect/aql/order.aql‎

Lines changed: 45 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -12,60 +12,65 @@ RETURN {
1212
SORT (v_root1.`name`)
1313
RETURN {
1414
"grandchildren": (
15+
LET v_root2 = NOEVAL({
16+
"name": v_root1.`name`
17+
})
1518
FOR v_item1
1619
IN v_root1.`children`[*].`children`[*][**]
1720
SORT (v_item1.`name`)
1821
RETURN {
1922
"name": v_item1.`name`,
20-
"root": {
21-
"name": v_root1.`name`
22-
}
23+
"root": v_root2
2324
}
2425
),
2526
"grandchildrenReverse": (
27+
LET v_root3 = NOEVAL({
28+
"name": v_root1.`name`
29+
})
2630
FOR v_item2
2731
IN v_root1.`children`[*].`children`[*][**]
2832
SORT (v_item2.`name`) DESC
2933
RETURN {
3034
"name": v_item2.`name`,
31-
"root": {
32-
"name": v_root1.`name`
33-
}
35+
"root": v_root3
3436
}
3537
),
3638
"extensionGrandchildren": (
39+
LET v_root4 = NOEVAL({
40+
"name": v_root1.`name`
41+
})
3742
FOR v_item3
3843
IN v_root1.`children`[*].`extension`.`children`[*][**]
3944
SORT (v_item3.`name`)
4045
RETURN {
4146
"name": v_item3.`name`,
42-
"root": {
43-
"name": v_root1.`name`
44-
}
47+
"root": v_root4
4548
}
4649
),
4750
"extensionGrandchildrenReverse": (
51+
LET v_root5 = NOEVAL({
52+
"name": v_root1.`name`
53+
})
4854
FOR v_item4
4955
IN v_root1.`children`[*].`extension`.`children`[*][**]
5056
SORT (v_item4.`name`) DESC
5157
RETURN {
5258
"name": v_item4.`name`,
53-
"root": {
54-
"name": v_root1.`name`
55-
}
59+
"root": v_root5
5660
}
5761
)
5862
}
5963
),
6064
"rootGrandchildren": (
61-
FOR v_root2, v_edge1, v_path1 IN @var1..@var2 OUTBOUND v_root21 @@root2s_roots
62-
FILTER v_root2._key != null
63-
FOR v_item5 IN v_root2.`children`[*].`children`[*][** RETURN {
65+
FOR v_root6, v_edge1, v_path1 IN @var1..@var2 OUTBOUND v_root21 @@root2s_roots
66+
FILTER v_root6._key != null
67+
LET v_root7 = NOEVAL({
68+
"name": v_root6.`name`
69+
})
70+
FOR v_item5 IN v_root6.`children`[*].`children`[*][** RETURN {
6471
value: {
6572
"name": CURRENT.`name`,
66-
"root": {
67-
"name": v_root2.`name`
68-
}
73+
"root": v_root7
6974
},
7075
sortValues: [
7176
CURRENT.`name`
@@ -75,14 +80,15 @@ RETURN {
7580
RETURN v_item5.value
7681
),
7782
"rootGrandchildrenReverse": (
78-
FOR v_root3, v_edge2, v_path2 IN @var3..@var4 OUTBOUND v_root21 @@root2s_roots
79-
FILTER v_root3._key != null
80-
FOR v_item6 IN v_root3.`children`[*].`children`[*][** RETURN {
83+
FOR v_root8, v_edge2, v_path2 IN @var3..@var4 OUTBOUND v_root21 @@root2s_roots
84+
FILTER v_root8._key != null
85+
LET v_root9 = NOEVAL({
86+
"name": v_root8.`name`
87+
})
88+
FOR v_item6 IN v_root8.`children`[*].`children`[*][** RETURN {
8189
value: {
8290
"name": CURRENT.`name`,
83-
"root": {
84-
"name": v_root3.`name`
85-
}
91+
"root": v_root9
8692
},
8793
sortValues: [
8894
CURRENT.`name`
@@ -92,14 +98,15 @@ RETURN {
9298
RETURN v_item6.value
9399
),
94100
"rootExtensionGrandchildren": (
95-
FOR v_root4, v_edge3, v_path3 IN @var5..@var6 OUTBOUND v_root21 @@root2s_roots
96-
FILTER v_root4._key != null
97-
FOR v_item7 IN v_root4.`children`[*].`extension`.`children`[*][** RETURN {
101+
FOR v_root10, v_edge3, v_path3 IN @var5..@var6 OUTBOUND v_root21 @@root2s_roots
102+
FILTER v_root10._key != null
103+
LET v_root11 = NOEVAL({
104+
"name": v_root10.`name`
105+
})
106+
FOR v_item7 IN v_root10.`children`[*].`extension`.`children`[*][** RETURN {
98107
value: {
99108
"name": CURRENT.`name`,
100-
"root": {
101-
"name": v_root4.`name`
102-
}
109+
"root": v_root11
103110
},
104111
sortValues: [
105112
CURRENT.`name`
@@ -109,14 +116,15 @@ RETURN {
109116
RETURN v_item7.value
110117
),
111118
"rootExtensionGrandchildrenReverse": (
112-
FOR v_root5, v_edge4, v_path4 IN @var7..@var8 OUTBOUND v_root21 @@root2s_roots
113-
FILTER v_root5._key != null
114-
FOR v_item8 IN v_root5.`children`[*].`extension`.`children`[*][** RETURN {
119+
FOR v_root12, v_edge4, v_path4 IN @var7..@var8 OUTBOUND v_root21 @@root2s_roots
120+
FILTER v_root12._key != null
121+
LET v_root13 = NOEVAL({
122+
"name": v_root12.`name`
123+
})
124+
FOR v_item8 IN v_root12.`children`[*].`extension`.`children`[*][** RETURN {
115125
value: {
116126
"name": CURRENT.`name`,
117-
"root": {
118-
"name": v_root5.`name`
119-
}
127+
"root": v_root13
120128
},
121129
sortValues: [
122130
CURRENT.`name`
@@ -129,4 +137,4 @@ RETURN {
129137
)
130138
}
131139

132-
// Peak memory usage: 1114112 bytes
140+
// Peak memory usage: 622592 bytes

0 commit comments

Comments
 (0)