From edc3ff4eee5a35e8eeb0427db5b5beccc5d19077 Mon Sep 17 00:00:00 2001 From: Paul Latzelsperger Date: Mon, 24 Jun 2024 13:01:59 +0200 Subject: [PATCH 1/5] let catalog extend dataset, update distribution resolver --- .../control-plane-catalog/build.gradle.kts | 3 +- .../catalog/DatasetResolverImpl.java | 3 +- .../catalog/DefaultDistributionResolver.java | 11 +++ .../DefaultDistributionResolverTest.java | 26 +++++++ .../controlplane/catalog/spi/Catalog.java | 67 ++++--------------- .../controlplane/catalog/spi/Dataset.java | 51 +++++++------- 6 files changed, 81 insertions(+), 80 deletions(-) diff --git a/core/control-plane/control-plane-catalog/build.gradle.kts b/core/control-plane/control-plane-catalog/build.gradle.kts index 89f9017feb2..e12fbfea557 100644 --- a/core/control-plane/control-plane-catalog/build.gradle.kts +++ b/core/control-plane/control-plane-catalog/build.gradle.kts @@ -22,7 +22,8 @@ dependencies { api(project(":spi:control-plane:transfer-spi")) api(project(":spi:control-plane:asset-spi")) - testImplementation(project(":tests:junit-base")); + implementation(project(":spi:common:data-address:data-address-http-data-spi")) + testImplementation(project(":tests:junit-base")) testImplementation(project(":core:common:connector-core")) testImplementation(project(":core:control-plane:control-plane-core")) diff --git a/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImpl.java b/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImpl.java index c9a4239ccec..b4496d9bfb0 100644 --- a/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImpl.java +++ b/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImpl.java @@ -16,6 +16,7 @@ import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset; import org.eclipse.edc.connector.controlplane.asset.spi.index.AssetIndex; +import org.eclipse.edc.connector.controlplane.catalog.spi.Catalog; import org.eclipse.edc.connector.controlplane.catalog.spi.Dataset; import org.eclipse.edc.connector.controlplane.catalog.spi.DatasetResolver; import org.eclipse.edc.connector.controlplane.catalog.spi.DistributionResolver; @@ -78,7 +79,7 @@ public Dataset getById(ParticipantAgent agent, String id) { private Dataset toDataset(List contractDefinitions, Asset asset) { var distributions = distributionResolver.getDistributions(asset); - var datasetBuilder = Dataset.Builder.newInstance() + var datasetBuilder = asset.isCatalog() ? Catalog.Builder.newInstance() : Dataset.Builder.newInstance() .id(asset.getId()) .distributions(distributions) .properties(asset.getProperties()); diff --git a/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DefaultDistributionResolver.java b/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DefaultDistributionResolver.java index c6d48f7fc64..813dd7bba2c 100644 --- a/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DefaultDistributionResolver.java +++ b/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DefaultDistributionResolver.java @@ -16,10 +16,12 @@ package org.eclipse.edc.connector.controlplane.catalog; import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset; +import org.eclipse.edc.connector.controlplane.catalog.spi.DataService; import org.eclipse.edc.connector.controlplane.catalog.spi.DataServiceRegistry; import org.eclipse.edc.connector.controlplane.catalog.spi.Distribution; import org.eclipse.edc.connector.controlplane.catalog.spi.DistributionResolver; import org.eclipse.edc.connector.controlplane.transfer.spi.flow.DataFlowManager; +import org.eclipse.edc.dataaddress.httpdata.spi.HttpDataAddressSchema; import java.util.List; @@ -35,6 +37,15 @@ public DefaultDistributionResolver(DataServiceRegistry dataServiceRegistry, Data @Override public List getDistributions(Asset asset) { + if (asset.isCatalog()) { + return List.of(Distribution.Builder.newInstance() + .format(asset.getDataAddress().getType()) + .dataService(DataService.Builder.newInstance() + .endpointDescription(asset.getDescription()) + .endpointUrl(asset.getDataAddress().getStringProperty(HttpDataAddressSchema.BASE_URL, null)) + .build()) + .build()); + } return dataFlowManager.transferTypesFor(asset).stream().map(this::createDistribution).toList(); } diff --git a/core/control-plane/control-plane-catalog/src/test/java/org/eclipse/edc/connector/controlplane/catalog/DefaultDistributionResolverTest.java b/core/control-plane/control-plane-catalog/src/test/java/org/eclipse/edc/connector/controlplane/catalog/DefaultDistributionResolverTest.java index 2b87eee063b..463d1b7600e 100644 --- a/core/control-plane/control-plane-catalog/src/test/java/org/eclipse/edc/connector/controlplane/catalog/DefaultDistributionResolverTest.java +++ b/core/control-plane/control-plane-catalog/src/test/java/org/eclipse/edc/connector/controlplane/catalog/DefaultDistributionResolverTest.java @@ -18,6 +18,7 @@ import org.eclipse.edc.connector.controlplane.catalog.spi.DataService; import org.eclipse.edc.connector.controlplane.catalog.spi.DataServiceRegistry; import org.eclipse.edc.connector.controlplane.transfer.spi.flow.DataFlowManager; +import org.eclipse.edc.dataaddress.httpdata.spi.HttpDataAddressSchema; import org.eclipse.edc.spi.types.domain.DataAddress; import org.junit.jupiter.api.Test; @@ -57,4 +58,29 @@ void shouldReturnDistributionsForEveryTransferType() { assertThat(distribution.getDataService()).isSameAs(dataService); }); } + + @Test + void shouldReturnDistribution_whenAssetIsCatalog() { + when(dataServiceRegistry.getDataServices()).thenReturn(List.of(dataService)); + when(dataFlowManager.transferTypesFor(any())).thenReturn(Set.of("type1", "type2")); + + var dataAddress = DataAddress.Builder.newInstance() + .type("HttpData") + .property(HttpDataAddressSchema.BASE_URL, "http://quizzqua.zz/buzz") + .build(); + var asset = Asset.Builder.newInstance() + .dataAddress(dataAddress) + .property(Asset.PROPERTY_IS_CATALOG, true) + .description("test description") + .build(); + + var distributions = resolver.getDistributions(asset); + + assertThat(distributions).hasSize(1) + .anySatisfy(distribution -> { + assertThat(distribution.getFormat()).isEqualTo("HttpData"); + assertThat(distribution.getDataService().getEndpointDescription()).isEqualTo(asset.getDescription()); + assertThat(distribution.getDataService().getEndpointUrl()).isEqualTo("http://quizzqua.zz/buzz"); + }); + } } diff --git a/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Catalog.java b/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Catalog.java index 03f4ef04d71..09e8742d8d0 100644 --- a/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Catalog.java +++ b/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Catalog.java @@ -19,26 +19,16 @@ import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; - -import static java.util.UUID.randomUUID; /** * Entity representing a Catalog */ @JsonDeserialize(builder = Catalog.Builder.class) -public class Catalog { - private String id; - private List datasets = new ArrayList<>(); - private List dataServices; - private Map properties; - private String participantId; - - public String getId() { - return id; - } +public class Catalog extends Dataset { + protected final List datasets = new ArrayList<>(); + protected List dataServices; + protected String participantId; public List getDatasets() { return datasets; @@ -48,78 +38,47 @@ public List getDataServices() { return dataServices; } - public Map getProperties() { - return properties; - } - public String getParticipantId() { return participantId; } @JsonPOJOBuilder(withPrefix = "") - public static class Builder { - private final Catalog catalog; + public static class Builder extends Dataset.Builder { private Builder() { - catalog = new Catalog(); + super(new Catalog()); } public static Builder newInstance() { return new Builder(); } - public Builder id(String id) { - catalog.id = id; - return this; - } - public Builder datasets(List datasets) { - catalog.datasets.addAll(datasets); + dataset.datasets.addAll(datasets); return this; } public Builder dataset(Dataset dataset) { - catalog.datasets.add(dataset); + this.dataset.datasets.add(dataset); return this; } public Builder dataServices(List dataServices) { - catalog.dataServices = dataServices; + this.dataset.dataServices = dataServices; return this; } public Builder dataService(DataService dataService) { - if (catalog.dataServices == null) { - catalog.dataServices = new ArrayList<>(); + if (this.dataset.dataServices == null) { + this.dataset.dataServices = new ArrayList<>(); } - catalog.dataServices.add(dataService); - return this; - } - - public Builder properties(Map properties) { - catalog.properties = properties; - return this; - } - - public Builder property(String key, Object value) { - if (catalog.properties == null) { - catalog.properties = new HashMap<>(); - } - catalog.properties.put(key, value); + this.dataset.dataServices.add(dataService); return this; } public Builder participantId(String participantId) { - catalog.participantId = participantId; + this.dataset.participantId = participantId; return this; } - - public Catalog build() { - if (catalog.id == null) { - catalog.id = randomUUID().toString(); - } - - return catalog; - } } } diff --git a/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Dataset.java b/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Dataset.java index 16c87a72588..60d506e6dec 100644 --- a/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Dataset.java +++ b/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Dataset.java @@ -35,22 +35,20 @@ @JsonDeserialize(builder = Dataset.Builder.class) public class Dataset { - private String id; - /** * Policies under which this Dataset is available. */ - private final Map offers = new HashMap<>(); - + protected final Map offers = new HashMap<>(); + protected String id; /** * Representations of this Dataset. */ - private List distributions; + protected List distributions; /** * Properties for describing the Dataset. */ - private Map properties = new HashMap<>(); + protected Map properties = new HashMap<>(); public String getId() { return id; @@ -78,61 +76,66 @@ public boolean hasOffers() { } @JsonPOJOBuilder(withPrefix = "") - public static class Builder { - private final Dataset dataset; + public static class Builder> { + protected final T dataset; - private Builder() { - dataset = new Dataset(); + protected Builder(T dataset) { + this.dataset = dataset; } + @SuppressWarnings({ "rawtypes", "unchecked" }) @JsonCreator public static Builder newInstance() { - return new Builder(); + return new Builder(new Dataset()); } - public Builder id(String id) { + public B id(String id) { dataset.id = id; - return this; + return self(); } - public Builder offer(String offerId, Policy policy) { + public B offer(String offerId, Policy policy) { dataset.offers.put(offerId, policy); - return this; + return self(); } - public Builder distribution(Distribution distribution) { + public B distribution(Distribution distribution) { if (dataset.distributions == null) { dataset.distributions = new ArrayList<>(); } dataset.distributions.add(distribution); - return this; + return self(); } - public Builder distributions(List distributions) { + public B distributions(List distributions) { if (dataset.distributions == null) { dataset.distributions = new ArrayList<>(); } dataset.distributions.addAll(distributions); - return this; + return self(); } - public Builder properties(Map properties) { + public B properties(Map properties) { dataset.properties = properties; - return this; + return self(); } - public Builder property(String key, Object value) { + public B property(String key, Object value) { dataset.properties.put(key, value); - return this; + return self(); } - public Dataset build() { + public T build() { if (dataset.id == null) { dataset.id = randomUUID().toString(); } return dataset; } + + public B self() { + return (B) this; + } } } From 5019ebf55d5eb6853cdca0037a6836f1acfc1dba Mon Sep 17 00:00:00 2001 From: Paul Latzelsperger Date: Mon, 24 Jun 2024 13:14:56 +0200 Subject: [PATCH 2/5] update dataset resolver + test --- .../catalog/DatasetResolverImpl.java | 3 +- .../catalog/DatasetResolverImplTest.java | 30 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImpl.java b/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImpl.java index b4496d9bfb0..60d17e4afd5 100644 --- a/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImpl.java +++ b/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImpl.java @@ -79,7 +79,8 @@ public Dataset getById(ParticipantAgent agent, String id) { private Dataset toDataset(List contractDefinitions, Asset asset) { var distributions = distributionResolver.getDistributions(asset); - var datasetBuilder = asset.isCatalog() ? Catalog.Builder.newInstance() : Dataset.Builder.newInstance() + var datasetBuilder = asset.isCatalog() ? Catalog.Builder.newInstance() : Dataset.Builder.newInstance(); + datasetBuilder .id(asset.getId()) .distributions(distributions) .properties(asset.getProperties()); diff --git a/core/control-plane/control-plane-catalog/src/test/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImplTest.java b/core/control-plane/control-plane-catalog/src/test/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImplTest.java index e7eacd26199..ec6807e0627 100644 --- a/core/control-plane/control-plane-catalog/src/test/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImplTest.java +++ b/core/control-plane/control-plane-catalog/src/test/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImplTest.java @@ -18,6 +18,7 @@ import org.assertj.core.api.iterable.ThrowingExtractor; import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset; import org.eclipse.edc.connector.controlplane.asset.spi.index.AssetIndex; +import org.eclipse.edc.connector.controlplane.catalog.spi.Catalog; import org.eclipse.edc.connector.controlplane.catalog.spi.DataService; import org.eclipse.edc.connector.controlplane.catalog.spi.Dataset; import org.eclipse.edc.connector.controlplane.catalog.spi.DatasetResolver; @@ -28,6 +29,7 @@ import org.eclipse.edc.connector.controlplane.contract.spi.types.offer.ContractDefinition; import org.eclipse.edc.connector.controlplane.policy.spi.PolicyDefinition; import org.eclipse.edc.connector.controlplane.policy.spi.store.PolicyDefinitionStore; +import org.eclipse.edc.dataaddress.httpdata.spi.HttpDataAddressSchema; import org.eclipse.edc.policy.model.Policy; import org.eclipse.edc.query.CriterionOperatorRegistryImpl; import org.eclipse.edc.spi.agent.ParticipantAgent; @@ -219,6 +221,34 @@ void query_shouldLimitDataset_whenMultipleDefinitionsWithSameAssets() { .map(getId()).containsExactly("6", "7"); } + @Test + void query_shouldReturnCatalogWithinCatalog_whenAssetIsCatalogAsset() { + var contractDefinition = contractDefinitionBuilder("definitionId").contractPolicyId("contractPolicyId").build(); + var contractPolicy = Policy.Builder.newInstance().build(); + var distribution = Distribution.Builder.newInstance().dataService(DataService.Builder.newInstance() + .endpointDescription("test-asset-desc") + .endpointUrl("https://foo.bar/baz") + .build()) + .format(HttpDataAddressSchema.HTTP_DATA_TYPE).build(); + + when(contractDefinitionResolver.definitionsFor(any())).thenReturn(Stream.of(contractDefinition)); + when(assetIndex.queryAssets(isA(QuerySpec.class))).thenReturn(Stream.of(createAsset("assetId").property(Asset.PROPERTY_IS_CATALOG, true).build())); + when(policyStore.findById("contractPolicyId")).thenReturn(PolicyDefinition.Builder.newInstance().policy(contractPolicy).build()); + when(distributionResolver.getDistributions(isA(Asset.class))).thenReturn(List.of(distribution)); + + var datasets = datasetResolver.query(createParticipantAgent(), QuerySpec.none()); + + assertThat(datasets).isNotNull().hasSize(1).first().satisfies(dataset -> { + assertThat(dataset).isInstanceOf(Catalog.class); + assertThat(dataset.getId()).isEqualTo("assetId"); + assertThat(dataset.getOffers()).hasSize(1).allSatisfy((id, policy) -> { + assertThat(ContractOfferId.parseId(id)).isSucceeded().extracting(ContractOfferId::definitionPart).asString().isEqualTo("definitionId"); + assertThat(policy.getType()).isEqualTo(OFFER); + assertThat(policy.getTarget()).isEqualTo(null); + }); + }); + } + @Test void getById_shouldReturnDataset() { var policy1 = Policy.Builder.newInstance().inheritsFrom("inherits1").build(); From 302b7b37d47d8fd000209075b6b16b24dd8c2318 Mon Sep 17 00:00:00 2001 From: Paul Latzelsperger Date: Tue, 25 Jun 2024 06:54:57 +0200 Subject: [PATCH 3/5] add e2e test (wip) --- .../controlplane/catalog/spi/Dataset.java | 2 +- .../managementapi/CatalogApiEndToEndTest.java | 71 ++++++++++++++++--- 2 files changed, 62 insertions(+), 11 deletions(-) diff --git a/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Dataset.java b/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Dataset.java index 60d506e6dec..0400220e6b7 100644 --- a/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Dataset.java +++ b/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Dataset.java @@ -43,7 +43,7 @@ public class Dataset { /** * Representations of this Dataset. */ - protected List distributions; + protected List distributions = new ArrayList<>(); /** * Properties for describing the Dataset. diff --git a/system-tests/management-api/management-api-test-runner/src/test/java/org/eclipse/edc/test/e2e/managementapi/CatalogApiEndToEndTest.java b/system-tests/management-api/management-api-test-runner/src/test/java/org/eclipse/edc/test/e2e/managementapi/CatalogApiEndToEndTest.java index 1d91a384ee2..2a9926f8893 100644 --- a/system-tests/management-api/management-api-test-runner/src/test/java/org/eclipse/edc/test/e2e/managementapi/CatalogApiEndToEndTest.java +++ b/system-tests/management-api/management-api-test-runner/src/test/java/org/eclipse/edc/test/e2e/managementapi/CatalogApiEndToEndTest.java @@ -14,6 +14,8 @@ package org.eclipse.edc.test.e2e.managementapi; +import jakarta.json.JsonObject; +import jakarta.json.JsonString; import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset; import org.eclipse.edc.connector.controlplane.asset.spi.index.AssetIndex; import org.eclipse.edc.connector.controlplane.contract.spi.offer.store.ContractDefinitionStore; @@ -35,6 +37,7 @@ import static io.restassured.http.ContentType.JSON; import static jakarta.json.Json.createArrayBuilder; import static jakarta.json.Json.createObjectBuilder; +import static org.assertj.core.api.Assertions.assertThat; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.CONTEXT; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.ID; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.TYPE; @@ -45,16 +48,6 @@ public class CatalogApiEndToEndTest { - @Nested - @EndToEndTest - @ExtendWith(ManagementEndToEndExtension.InMemory.class) - class InMemory extends Tests { } - - @Nested - @PostgresqlIntegrationTest - @ExtendWith(ManagementEndToEndExtension.Postgres.class) - class Postgres extends Tests { } - abstract static class Tests { @Test @@ -132,6 +125,51 @@ void requestCatalog_shouldReturnCatalog_withQuerySpec(ManagementEndToEndTestCont .body("'dcat:dataset'.id", is("id-2")); } + @Test + void requestCatalog_whenAssetIsCatalogAsset_shouldReturnCatalogOfCatalogs(ManagementEndToEndTestContext context, AssetIndex assetIndex, + PolicyDefinitionStore policyDefinitionStore, + ContractDefinitionStore contractDefinitionStore) { + var policyId = UUID.randomUUID().toString(); + + var cd = ContractDefinition.Builder.newInstance() + .id(UUID.randomUUID().toString()) + .contractPolicyId(policyId) + .accessPolicyId(policyId) + .build(); + + var policy = Policy.Builder.newInstance() + .build(); + + policyDefinitionStore.create(PolicyDefinition.Builder.newInstance().id(policyId).policy(policy).build()); + contractDefinitionStore.save(cd); + + var httpData = createAsset("id-1", "HttpData") + .property(Asset.PROPERTY_IS_CATALOG, true) + .build(); + httpData.getDataAddress().getProperties().put(EDC_NAMESPACE + "baseUrl", "http://quizzqua.zz/buzz"); + assetIndex.create(httpData); + + var requestBody = createObjectBuilder() + .add(CONTEXT, createObjectBuilder().add(EDC_PREFIX, EDC_NAMESPACE)) + .add(TYPE, "CatalogRequest") + .add("counterPartyAddress", context.providerProtocolUrl()) + .add("protocol", "dataspace-protocol-http") + .build(); + + var body = context.baseRequest() + .contentType(JSON) + .body(requestBody) + .post("/v3/catalog/request") + .then() + .statusCode(200) + .contentType(JSON) + .extract().body().as(JsonObject.class); + + var actual = body.get(TYPE); + assertThat(actual).isInstanceOf(JsonString.class); + assertThat(((JsonString) actual).getString()).isEqualTo("dcat:Catalog"); + } + @Test void getDataset_shouldReturnDataset(ManagementEndToEndTestContext context, AssetIndex assetIndex, DataPlaneInstanceStore dataPlaneInstanceStore) { @@ -161,6 +199,7 @@ void getDataset_shouldReturnDataset(ManagementEndToEndTestContext context, Asset .body("'dcat:distribution'.'dcat:accessService'.@id", notNullValue()); } + private Asset.Builder createAsset(String id, String sourceType) { return Asset.Builder.newInstance() .dataAddress(DataAddress.Builder.newInstance().type(sourceType).build()) @@ -169,4 +208,16 @@ private Asset.Builder createAsset(String id, String sourceType) { } + @Nested + @EndToEndTest + @ExtendWith(ManagementEndToEndExtension.InMemory.class) + class InMemory extends Tests { + } + + @Nested + @PostgresqlIntegrationTest + @ExtendWith(ManagementEndToEndExtension.Postgres.class) + class Postgres extends Tests { + } + } From f3d774bb7c8f6d20834996e8477305f653bc2bb3 Mon Sep 17 00:00:00 2001 From: Paul Latzelsperger Date: Tue, 25 Jun 2024 13:03:22 +0200 Subject: [PATCH 4/5] added e2e test --- .../catalog/DatasetResolverImpl.java | 19 ++++++- .../catalog/DefaultDistributionResolver.java | 5 +- .../catalog/DatasetResolverImplTest.java | 6 ++- .../DefaultDistributionResolverTest.java | 3 +- .../JsonObjectFromCatalogTransformer.java | 10 +++- .../controlplane/catalog/spi/Catalog.java | 2 +- .../catalog/spi/Distribution.java | 1 - .../managementapi/CatalogApiEndToEndTest.java | 52 ++++++++++++------- 8 files changed, 67 insertions(+), 31 deletions(-) diff --git a/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImpl.java b/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImpl.java index 60d17e4afd5..e0b04eff13a 100644 --- a/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImpl.java +++ b/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImpl.java @@ -17,6 +17,7 @@ import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset; import org.eclipse.edc.connector.controlplane.asset.spi.index.AssetIndex; import org.eclipse.edc.connector.controlplane.catalog.spi.Catalog; +import org.eclipse.edc.connector.controlplane.catalog.spi.DataService; import org.eclipse.edc.connector.controlplane.catalog.spi.Dataset; import org.eclipse.edc.connector.controlplane.catalog.spi.DatasetResolver; import org.eclipse.edc.connector.controlplane.catalog.spi.DistributionResolver; @@ -24,12 +25,14 @@ import org.eclipse.edc.connector.controlplane.contract.spi.offer.ContractDefinitionResolver; import org.eclipse.edc.connector.controlplane.contract.spi.types.offer.ContractDefinition; import org.eclipse.edc.connector.controlplane.policy.spi.store.PolicyDefinitionStore; +import org.eclipse.edc.dataaddress.httpdata.spi.HttpDataAddressSchema; import org.eclipse.edc.policy.model.PolicyType; import org.eclipse.edc.spi.agent.ParticipantAgent; import org.eclipse.edc.spi.query.CriterionOperatorRegistry; import org.eclipse.edc.spi.query.QuerySpec; import org.jetbrains.annotations.NotNull; +import java.util.Base64; import java.util.List; import java.util.Optional; import java.util.function.Predicate; @@ -76,11 +79,23 @@ public Dataset getById(ParticipantAgent agent, String id) { .orElse(null); } + private Dataset.Builder createCatalog(Asset asset) { + if (!asset.isCatalog()) { + return Dataset.Builder.newInstance(); + } + + return Catalog.Builder.newInstance() + .dataService(DataService.Builder.newInstance() + .id(Base64.getUrlEncoder().encodeToString(asset.getId().getBytes())) + .endpointDescription(asset.getDescription()) + .endpointUrl(asset.getDataAddress().getStringProperty(HttpDataAddressSchema.BASE_URL, null)) + .build()); + } + private Dataset toDataset(List contractDefinitions, Asset asset) { var distributions = distributionResolver.getDistributions(asset); - var datasetBuilder = asset.isCatalog() ? Catalog.Builder.newInstance() : Dataset.Builder.newInstance(); - datasetBuilder + var datasetBuilder = createCatalog(asset) .id(asset.getId()) .distributions(distributions) .properties(asset.getProperties()); diff --git a/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DefaultDistributionResolver.java b/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DefaultDistributionResolver.java index 813dd7bba2c..d2f41d6d6ec 100644 --- a/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DefaultDistributionResolver.java +++ b/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DefaultDistributionResolver.java @@ -21,8 +21,8 @@ import org.eclipse.edc.connector.controlplane.catalog.spi.Distribution; import org.eclipse.edc.connector.controlplane.catalog.spi.DistributionResolver; import org.eclipse.edc.connector.controlplane.transfer.spi.flow.DataFlowManager; -import org.eclipse.edc.dataaddress.httpdata.spi.HttpDataAddressSchema; +import java.util.Base64; import java.util.List; public class DefaultDistributionResolver implements DistributionResolver { @@ -41,8 +41,7 @@ public List getDistributions(Asset asset) { return List.of(Distribution.Builder.newInstance() .format(asset.getDataAddress().getType()) .dataService(DataService.Builder.newInstance() - .endpointDescription(asset.getDescription()) - .endpointUrl(asset.getDataAddress().getStringProperty(HttpDataAddressSchema.BASE_URL, null)) + .id(Base64.getUrlEncoder().encodeToString(asset.getId().getBytes())) .build()) .build()); } diff --git a/core/control-plane/control-plane-catalog/src/test/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImplTest.java b/core/control-plane/control-plane-catalog/src/test/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImplTest.java index ec6807e0627..ab3f5e7d128 100644 --- a/core/control-plane/control-plane-catalog/src/test/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImplTest.java +++ b/core/control-plane/control-plane-catalog/src/test/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImplTest.java @@ -36,6 +36,7 @@ import org.eclipse.edc.spi.message.Range; import org.eclipse.edc.spi.query.Criterion; import org.eclipse.edc.spi.query.QuerySpec; +import org.eclipse.edc.spi.types.domain.DataAddress; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -232,7 +233,10 @@ void query_shouldReturnCatalogWithinCatalog_whenAssetIsCatalogAsset() { .format(HttpDataAddressSchema.HTTP_DATA_TYPE).build(); when(contractDefinitionResolver.definitionsFor(any())).thenReturn(Stream.of(contractDefinition)); - when(assetIndex.queryAssets(isA(QuerySpec.class))).thenReturn(Stream.of(createAsset("assetId").property(Asset.PROPERTY_IS_CATALOG, true).build())); + when(assetIndex.queryAssets(isA(QuerySpec.class))).thenReturn(Stream.of(createAsset("assetId") + .property(Asset.PROPERTY_IS_CATALOG, true) + .dataAddress(DataAddress.Builder.newInstance().type(HttpDataAddressSchema.HTTP_DATA_TYPE).build()) + .build())); when(policyStore.findById("contractPolicyId")).thenReturn(PolicyDefinition.Builder.newInstance().policy(contractPolicy).build()); when(distributionResolver.getDistributions(isA(Asset.class))).thenReturn(List.of(distribution)); diff --git a/core/control-plane/control-plane-catalog/src/test/java/org/eclipse/edc/connector/controlplane/catalog/DefaultDistributionResolverTest.java b/core/control-plane/control-plane-catalog/src/test/java/org/eclipse/edc/connector/controlplane/catalog/DefaultDistributionResolverTest.java index 463d1b7600e..8b39a601e99 100644 --- a/core/control-plane/control-plane-catalog/src/test/java/org/eclipse/edc/connector/controlplane/catalog/DefaultDistributionResolverTest.java +++ b/core/control-plane/control-plane-catalog/src/test/java/org/eclipse/edc/connector/controlplane/catalog/DefaultDistributionResolverTest.java @@ -79,8 +79,7 @@ void shouldReturnDistribution_whenAssetIsCatalog() { assertThat(distributions).hasSize(1) .anySatisfy(distribution -> { assertThat(distribution.getFormat()).isEqualTo("HttpData"); - assertThat(distribution.getDataService().getEndpointDescription()).isEqualTo(asset.getDescription()); - assertThat(distribution.getDataService().getEndpointUrl()).isEqualTo("http://quizzqua.zz/buzz"); + assertThat(distribution.getDataService().getId()).isNotNull(); }); } } diff --git a/data-protocols/dsp/dsp-catalog/dsp-catalog-transform/src/main/java/org/eclipse/edc/protocol/dsp/catalog/transform/from/JsonObjectFromCatalogTransformer.java b/data-protocols/dsp/dsp-catalog/dsp-catalog-transform/src/main/java/org/eclipse/edc/protocol/dsp/catalog/transform/from/JsonObjectFromCatalogTransformer.java index c8e6f86f51a..26bb581be3c 100644 --- a/data-protocols/dsp/dsp-catalog/dsp-catalog-transform/src/main/java/org/eclipse/edc/protocol/dsp/catalog/transform/from/JsonObjectFromCatalogTransformer.java +++ b/data-protocols/dsp/dsp-catalog/dsp-catalog-transform/src/main/java/org/eclipse/edc/protocol/dsp/catalog/transform/from/JsonObjectFromCatalogTransformer.java @@ -25,11 +25,13 @@ import org.jetbrains.annotations.Nullable; import static jakarta.json.stream.JsonCollectors.toJsonArray; +import static java.util.Optional.ofNullable; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.ID; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.TYPE; import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_CATALOG_TYPE; import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_DATASET_ATTRIBUTE; import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_DATA_SERVICE_ATTRIBUTE; +import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DCAT_DISTRIBUTION_ATTRIBUTE; import static org.eclipse.edc.jsonld.spi.PropertyAndTypeNames.DSPACE_PROPERTY_PARTICIPANT_ID; /** @@ -57,13 +59,19 @@ public JsonObjectFromCatalogTransformer(JsonBuilderFactory jsonFactory, ObjectMa .map(service -> context.transform(service, JsonObject.class)) .collect(toJsonArray()); + var distributions = catalog.getDistributions().stream() + .map(distro -> context.transform(distro, JsonObject.class)) + .collect(toJsonArray()); + var objectBuilder = jsonFactory.createObjectBuilder() .add(ID, catalog.getId()) .add(TYPE, DCAT_CATALOG_TYPE) - .add(DSPACE_PROPERTY_PARTICIPANT_ID, participantIdMapper.toIri(catalog.getParticipantId())) .add(DCAT_DATASET_ATTRIBUTE, datasets) + .add(DCAT_DISTRIBUTION_ATTRIBUTE, distributions) .add(DCAT_DATA_SERVICE_ATTRIBUTE, dataServices); + ofNullable(catalog.getParticipantId()).ifPresent(pid -> objectBuilder.add(DSPACE_PROPERTY_PARTICIPANT_ID, participantIdMapper.toIri(pid))); + transformProperties(catalog.getProperties(), objectBuilder, mapper, context); return objectBuilder.build(); diff --git a/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Catalog.java b/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Catalog.java index 09e8742d8d0..fe48f6f27ee 100644 --- a/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Catalog.java +++ b/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Catalog.java @@ -27,7 +27,7 @@ @JsonDeserialize(builder = Catalog.Builder.class) public class Catalog extends Dataset { protected final List datasets = new ArrayList<>(); - protected List dataServices; + protected List dataServices = new ArrayList<>(); protected String participantId; public List getDatasets() { diff --git a/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Distribution.java b/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Distribution.java index dcfce81c43e..04361859efe 100644 --- a/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Distribution.java +++ b/spi/control-plane/catalog-spi/src/main/java/org/eclipse/edc/connector/controlplane/catalog/spi/Distribution.java @@ -71,7 +71,6 @@ public Builder dataService(DataService dataService) { public Distribution build() { Objects.requireNonNull(distribution.dataService, "DataService must not be null"); - Objects.requireNonNull(distribution.format, "Format must not be null"); return distribution; } diff --git a/system-tests/management-api/management-api-test-runner/src/test/java/org/eclipse/edc/test/e2e/managementapi/CatalogApiEndToEndTest.java b/system-tests/management-api/management-api-test-runner/src/test/java/org/eclipse/edc/test/e2e/managementapi/CatalogApiEndToEndTest.java index 2a9926f8893..f1b08b6d71e 100644 --- a/system-tests/management-api/management-api-test-runner/src/test/java/org/eclipse/edc/test/e2e/managementapi/CatalogApiEndToEndTest.java +++ b/system-tests/management-api/management-api-test-runner/src/test/java/org/eclipse/edc/test/e2e/managementapi/CatalogApiEndToEndTest.java @@ -14,8 +14,6 @@ package org.eclipse.edc.test.e2e.managementapi; -import jakarta.json.JsonObject; -import jakarta.json.JsonString; import org.eclipse.edc.connector.controlplane.asset.spi.domain.Asset; import org.eclipse.edc.connector.controlplane.asset.spi.index.AssetIndex; import org.eclipse.edc.connector.controlplane.contract.spi.offer.store.ContractDefinitionStore; @@ -27,23 +25,27 @@ import org.eclipse.edc.junit.annotations.EndToEndTest; import org.eclipse.edc.junit.annotations.PostgresqlIntegrationTest; import org.eclipse.edc.policy.model.Policy; +import org.eclipse.edc.spi.query.Criterion; import org.eclipse.edc.spi.types.domain.DataAddress; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import java.util.Base64; +import java.util.List; import java.util.UUID; import static io.restassured.http.ContentType.JSON; import static jakarta.json.Json.createArrayBuilder; import static jakarta.json.Json.createObjectBuilder; -import static org.assertj.core.api.Assertions.assertThat; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.CONTEXT; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.ID; import static org.eclipse.edc.jsonld.spi.JsonLdKeywords.TYPE; import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE; import static org.eclipse.edc.spi.constants.CoreConstants.EDC_PREFIX; import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; public class CatalogApiEndToEndTest { @@ -129,26 +131,33 @@ void requestCatalog_shouldReturnCatalog_withQuerySpec(ManagementEndToEndTestCont void requestCatalog_whenAssetIsCatalogAsset_shouldReturnCatalogOfCatalogs(ManagementEndToEndTestContext context, AssetIndex assetIndex, PolicyDefinitionStore policyDefinitionStore, ContractDefinitionStore contractDefinitionStore) { + // create and store policy var policyId = UUID.randomUUID().toString(); + var policy = Policy.Builder.newInstance().build(); + policyDefinitionStore.create(PolicyDefinition.Builder.newInstance().id(policyId).policy(policy).build()); + + // create CatalogAsset + var catalogAssetId = "catalog-asset-" + UUID.randomUUID(); + var httpData = createAsset(catalogAssetId, "HttpData") + .property(Asset.PROPERTY_IS_CATALOG, true) + .build(); + httpData.getDataAddress().getProperties().put(EDC_NAMESPACE + "baseUrl", "http://quizzqua.zz/buzz"); + assetIndex.create(httpData); + + // create conventional asset + var normalAssetId = "normal-asset-" + UUID.randomUUID(); + assetIndex.create(createAsset(normalAssetId, "test-type").build()); + // create ContractDefinition var cd = ContractDefinition.Builder.newInstance() .id(UUID.randomUUID().toString()) .contractPolicyId(policyId) .accessPolicyId(policyId) + .assetsSelector(List.of(Criterion.criterion("id", "in", List.of(catalogAssetId, normalAssetId)))) .build(); - - var policy = Policy.Builder.newInstance() - .build(); - - policyDefinitionStore.create(PolicyDefinition.Builder.newInstance().id(policyId).policy(policy).build()); contractDefinitionStore.save(cd); - var httpData = createAsset("id-1", "HttpData") - .property(Asset.PROPERTY_IS_CATALOG, true) - .build(); - httpData.getDataAddress().getProperties().put(EDC_NAMESPACE + "baseUrl", "http://quizzqua.zz/buzz"); - assetIndex.create(httpData); - + // request all assets var requestBody = createObjectBuilder() .add(CONTEXT, createObjectBuilder().add(EDC_PREFIX, EDC_NAMESPACE)) .add(TYPE, "CatalogRequest") @@ -156,18 +165,21 @@ void requestCatalog_whenAssetIsCatalogAsset_shouldReturnCatalogOfCatalogs(Manage .add("protocol", "dataspace-protocol-http") .build(); - var body = context.baseRequest() + context.baseRequest() .contentType(JSON) .body(requestBody) .post("/v3/catalog/request") .then() .statusCode(200) .contentType(JSON) - .extract().body().as(JsonObject.class); - - var actual = body.get(TYPE); - assertThat(actual).isInstanceOf(JsonString.class); - assertThat(((JsonString) actual).getString()).isEqualTo("dcat:Catalog"); + .body(TYPE, is("dcat:Catalog")) + .body("'dcat:service'", notNullValue()) + // findAll is the restAssured way to express JSON Path filters + .body("'dcat:dataset'", hasSize(2)) + .body("'dcat:dataset'.findAll { it -> it.'@type' == 'dcat:Catalog' }.isCatalog", contains(true)) + .body("'dcat:dataset'.findAll { it -> it.'@type' == 'dcat:Catalog' }.'@id'", contains(catalogAssetId)) + .body("'dcat:dataset'.findAll { it -> it.'@type' == 'dcat:Catalog' }.'dcat:service'.'dcat:endpointUrl'", contains("http://quizzqua.zz/buzz")) + .body("'dcat:dataset'.findAll { it -> it.'@type' == 'dcat:Catalog' }.'dcat:distribution'.'dcat:accessService'.'@id'", contains(Base64.getUrlEncoder().encodeToString(catalogAssetId.getBytes()))); } @Test From b0c53dc542ba016d1e799dffc8cff45a50e8bc02 Mon Sep 17 00:00:00 2001 From: Paul Latzelsperger Date: Tue, 25 Jun 2024 13:32:56 +0200 Subject: [PATCH 5/5] renamed method --- .../connector/controlplane/catalog/DatasetResolverImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImpl.java b/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImpl.java index e0b04eff13a..45378cff31f 100644 --- a/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImpl.java +++ b/core/control-plane/control-plane-catalog/src/main/java/org/eclipse/edc/connector/controlplane/catalog/DatasetResolverImpl.java @@ -79,7 +79,7 @@ public Dataset getById(ParticipantAgent agent, String id) { .orElse(null); } - private Dataset.Builder createCatalog(Asset asset) { + private Dataset.Builder buildDataset(Asset asset) { if (!asset.isCatalog()) { return Dataset.Builder.newInstance(); } @@ -95,7 +95,7 @@ private Dataset.Builder createCatalog(Asset asset) { private Dataset toDataset(List contractDefinitions, Asset asset) { var distributions = distributionResolver.getDistributions(asset); - var datasetBuilder = createCatalog(asset) + var datasetBuilder = buildDataset(asset) .id(asset.getId()) .distributions(distributions) .properties(asset.getProperties());