Skip to content

Commit 215e716

Browse files
mchrzamchrza
committed
DS-1723,DS-1841:
* rename connector psql-new-table-layout to nl-connector. * add new connector "psql-nl-connector". * automatically transform searchableProperties into new definition. * store new searchable definition in dynamo. Signed-off-by: mchrza <[email protected]>
1 parent a32dcd7 commit 215e716

File tree

8 files changed

+107
-54
lines changed

8 files changed

+107
-54
lines changed

‎xyz-hub-service/src/main/resources/connectors.json‎

Lines changed: 50 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
[
22
{
3-
"id": "psql-new-table-layout",
3+
"id": "nl-connector",
44
"params": {
5+
"autoIndexing": true,
56
"propertySearch": true,
67
"onDemandIdxLimit": 8,
78
"dbCheckoutTimeout": 12,
@@ -17,6 +18,9 @@
1718
"enableAutoCache": true,
1819
"maxPayloadSize": 1073741824,
1920
"storageUtilizationReporting": true,
21+
"clusteringTypes": [
22+
"hexbin", "quadbin"
23+
],
2024
"mvtSupport": true,
2125
"extensionSupport": true
2226
},
@@ -26,7 +30,7 @@
2630
"remoteFunctions": {
2731
"local": {
2832
"type": "Embedded",
29-
"id": "xyz-psql-local",
33+
"id": "nl-connector",
3034
"warmUp": 1,
3135
"className": "com.here.xyz.psql.NLConnector",
3236
"env": {
@@ -42,6 +46,49 @@
4246
"queryParams": ["apiKey"]
4347
}
4448
},
49+
{
50+
"id": "psql-nl-connector",
51+
"params": {
52+
"propertySearch": true,
53+
"onDemandIdxLimit": 8,
54+
"dbCheckoutTimeout": 12,
55+
"dbAcquireRetryAttempts": 10,
56+
"connectorId" : "psql",
57+
"ecps": "$encrypt({\"PSQL_HOST\":\"${PSQL_HOST}\",\"PSQL_REPLICA_HOST\":\"${PSQL_HOST}\",\"PSQL_REPLICA_USER\":\"ro_user\",\"PSQL_PORT\":\"${PSQL_PORT}\",\"PSQL_DB\":\"${PSQL_DB}\",\"PSQL_USER\":\"${PSQL_USER}\",\"PSQL_PASSWORD\":\"${PSQL_PASSWORD}\"})",
58+
"mvtSupport": true
59+
},
60+
"capabilities": {
61+
"propertySearch": true,
62+
"searchablePropertiesConfiguration": true,
63+
"preserializedResponseSupport": true,
64+
"enableAutoCache": true,
65+
"maxPayloadSize": 1073741824,
66+
"storageUtilizationReporting": true,
67+
"mvtSupport": true,
68+
"extensionSupport": true
69+
},
70+
"connectionSettings": {
71+
"maxConnections": 32
72+
},
73+
"remoteFunctions": {
74+
"local": {
75+
"type": "Embedded",
76+
"id": "psql-nl-connector",
77+
"warmUp": 1,
78+
"className": "com.here.xyz.psql.PSQLXyzNLConnector",
79+
"env": {
80+
"ECPS_PHRASE": "local"
81+
},
82+
"protocolVersion": "0.7.0"
83+
}
84+
},
85+
"trusted": true,
86+
"forwardParamsConfig": {
87+
"cookies": ["here_access", "here_access_st"],
88+
"headers": ["AuthorizatioN", "realm"],
89+
"queryParams": ["apiKey"]
90+
}
91+
},
4592
{
4693
"id": "psql",
4794
"params": {
@@ -75,7 +122,7 @@
75122
"type": "Embedded",
76123
"id": "xyz-psql-local",
77124
"warmUp": 1,
78-
"className": "com.here.xyz.psql.PSQLXyzNLConnector",
125+
"className": "com.here.xyz.psql.PSQLXyzConnector",
79126
"env": {
80127
"ECPS_PHRASE": "local"
81128
},

‎xyz-hub-test/src/test/java/com/here/xyz/hub/auth/AuthTestsIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -785,8 +785,8 @@ public void testCreateSpaceWithSearchableProperties() {
785785

786786
response
787787
.statusCode(OK.code())
788-
.body("searchableProperties.name", equalTo(true))
789-
.body("searchableProperties.other", equalTo(false));
788+
.body("searchableProperties['$name:[$.name]::scalar']", equalTo(true))
789+
.body("searchableProperties['$other:[$.other]::scalar']", equalTo(false));
790790
}
791791

792792
@Test

‎xyz-hub-test/src/test/java/com/here/xyz/hub/connectors/psqlnlconnector/ConnectorModifySpaceTest.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,11 @@ public void testCreateSpaceWithSearchableProperties() throws Exception {
2525
String spaceName = this.getClass().getSimpleName() +"."+Random.randomAlpha(6);
2626

2727
Map<String, Boolean> searchableProperties = Map.of(
28+
"foo1.nested", true,
2829
"$alias1:[$.properties.refQuad]::scalar", true,
2930
"$alias2:[$.properties.globalVersion]::scalar", true,
30-
"$alias3:[$.properties.names[*].lang]::array", true,
31-
"$foo1.nested:[$.foo1.nested]::scalar", true
31+
"$alias3:[$.properties.names[*].lang]::array", true
32+
//"$foo1.nested:[$.foo1.nested]::scalar", true
3233
// "$alias1:[$.properties.street.fc]::scalar", true,
3334
// "$alias2:[$.properties.names[*].lang]::array", true,
3435
// "$alias3:[$.properties.refQuad like_regex \"^0123\"]::scalar", true,

‎xyz-hub-test/src/test/java/com/here/xyz/hub/rest/CreateSpaceApiIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,8 +119,8 @@ public void createSpaceWithSearchablePropertiesPositive() {
119119

120120
response
121121
.statusCode(OK.code())
122-
.body("searchableProperties.name", equalTo(true))
123-
.body("searchableProperties.other", equalTo(false));
122+
.body("searchableProperties['$name:[$.name]::scalar']", equalTo(true))
123+
.body("searchableProperties['$other:[$.other]::scalar']", equalTo(false));
124124
}
125125

126126
@Test

‎xyz-hub-test/src/test/java/com/here/xyz/hub/rest/ModifySpaceApiIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,8 @@ public void setSearchablePropertiesPositive() {
6363
.patch("/spaces/x-psql-test")
6464
.then()
6565
.statusCode(OK.code())
66-
.body("searchableProperties.name", equalTo(true))
67-
.body("searchableProperties.other", equalTo(false));
66+
.body("searchableProperties['$name:[$.name]::scalar']", equalTo(true))
67+
.body("searchableProperties['$other:[$.other]::scalar']", equalTo(false));
6868
}
6969

7070
@Test

‎xyz-models/src/main/java/com/here/xyz/models/hub/Space.java‎

Lines changed: 6 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -530,60 +530,25 @@ public Space withReadOnlyHeadVersion(long readOnlyHeadVersion) {
530530
return this;
531531
}
532532

533-
// -------- Internal canonical accessors (used by server-side code) --------
534-
@JsonIgnore
535533
public Map<String, Boolean> getSearchableProperties() {
536534
return searchableProperties;
537535
}
538536

539537
@JsonIgnore
540538
public void setSearchableProperties(final Map<String, Boolean> searchableProperties) {
541-
this.searchableProperties = searchableProperties;
539+
if (searchableProperties == null || searchableProperties.isEmpty()) {
540+
this.searchableProperties = searchableProperties;
541+
return;
542+
}
543+
544+
this.searchableProperties = normalizeSearchableProperties(searchableProperties);
542545
}
543546

544547
public Space withSearchableProperties(final Map<String, Boolean> searchableProperties) {
545548
setSearchableProperties(searchableProperties);
546549
return this;
547550
}
548551

549-
// -------- JSON-facing alias accessors --------
550-
551-
@JsonInclude(Include.NON_EMPTY)
552-
@JsonView({Public.class, Static.class})
553-
@JsonProperty("searchableProperties")
554-
public Map<String, Boolean> getSearchablePropertiesByAlias() {
555-
if (searchableProperties == null || searchableProperties.isEmpty())
556-
return null;
557-
558-
Map<String, Boolean> byAlias = new LinkedHashMap<>();
559-
560-
for (Map.Entry<String, Boolean> e : searchableProperties.entrySet()) {
561-
String key = e.getKey();
562-
Boolean value = e.getValue();
563-
564-
try {
565-
NormalizedProperty np = parseAndNormalizeKey(key);
566-
byAlias.put(np.alias, value);
567-
}
568-
catch (IllegalArgumentException ex) {
569-
// In case of unexpected format, fallback
570-
byAlias.put(key, value);
571-
}
572-
}
573-
574-
return byAlias;
575-
}
576-
577-
@JsonProperty("searchableProperties")
578-
public void setSearchablePropertiesByAlias(final Map<String, Boolean> searchablePropertiesByAlias) {
579-
if (searchablePropertiesByAlias == null || searchablePropertiesByAlias.isEmpty()) {
580-
this.searchableProperties = searchablePropertiesByAlias;
581-
return;
582-
}
583-
584-
this.searchableProperties = normalizeSearchableProperties(searchablePropertiesByAlias);
585-
}
586-
587552
public List<List<Object>> getSortableProperties() {
588553
return sortableProperties;
589554
}

‎xyz-util/src/main/java/com/here/xyz/util/db/ConnectorParameters.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public class ConnectorParameters {
4242
public enum TableLayout {
4343
OLD_LAYOUT(false),
4444
OLD_LAYOUT_WITH_SEARCHABLE(true),
45-
NEW_LAYOUT(false);
45+
NEW_LAYOUT(true);
4646

4747
private final boolean hasSearchableColumn;
4848

‎xyz-util/src/main/java/com/here/xyz/util/db/pg/IndexHelper.java‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ public OnDemandIndex withIndexName(String indexName) {
135135

136136
public void setPropertyPath(String propertyPath) {
137137
this.propertyPath = propertyPath;
138+
transformIntoNewDefinition();
138139
}
139140

140141
public String getPropertyPath() {
@@ -196,6 +197,45 @@ public String extractLogicalPropertyPath() {
196197
return propertyPath;
197198
}
198199

200+
public void transformIntoNewDefinition() {
201+
//if already in new format, return as is
202+
if(propertyPath.startsWith("$"))
203+
return;
204+
205+
// Trim whitespace
206+
propertyPath = propertyPath.trim();
207+
208+
String path = propertyPath;
209+
String datatype = "scalar"; // default
210+
211+
// Check if datatype explicitly provided
212+
int idx = propertyPath.indexOf("::");
213+
if (idx >= 0) {
214+
path = propertyPath.substring(0, idx);
215+
String dt = propertyPath.substring(idx + 2).trim();
216+
if ("array".equalsIgnoreCase(dt)) {
217+
datatype = "array";
218+
}
219+
}
220+
221+
propertyPath = "$" + path + ":[$." + path + "]::" + datatype;
222+
}
223+
224+
public boolean definitionGotTransformed() {
225+
propertyPath = propertyPath.trim();
226+
int colonIndex = propertyPath.indexOf(':');
227+
228+
String leftPath = propertyPath.substring(1, colonIndex);
229+
230+
int openBracket = propertyPath.indexOf("[$.");
231+
int closeBracket = propertyPath.indexOf("]", openBracket);
232+
233+
String inner = propertyPath.substring(openBracket + 3, closeBracket);
234+
235+
// Compare paths
236+
return leftPath.equals(inner);
237+
}
238+
199239
@Override
200240
public String getIndexName(String tableName) {
201241
// Take the first 8 characters of md5 hash of the property path
@@ -246,7 +286,7 @@ public static SQLQuery buildOnDemandIndexCreationQuery(String schema, String tab
246286
if(layout.hasSearchableColumn())
247287
return buildOnDemandIndexCreationQuery(schema, table, index.extractAlias(), NEW_LAYOUT_INDEX_COULMN, async);
248288

249-
return buildOnDemandIndexCreationQuery(schema, table, index.getPropertyPath(), OLD_LAYOUT_INDEX_COULMN, async);
289+
return buildOnDemandIndexCreationQuery(schema, table, index.extractLogicalPropertyPath(), OLD_LAYOUT_INDEX_COULMN, async);
250290
}
251291

252292
public static SQLQuery buildOnDemandIndexCreationQuery(String schema, String table, String propertyPath, String targetColumn, boolean async){

0 commit comments

Comments
 (0)