From 9491659a64792ff56f968cb706387466bc4abcd0 Mon Sep 17 00:00:00 2001 From: Joost van der Griendt Date: Mon, 14 Nov 2022 16:57:29 +0100 Subject: [PATCH 01/41] add crossplane multicloud psql package --- multicloud/crossplane/psql/README.md | 14 ++ .../psql/claim-examples/helm-psql-12.yaml | 16 ++ multicloud/crossplane/psql/claim/helm.yml | 18 ++ .../psql/wip/aws-composition.ytt.yml | 111 +++++++++++ .../psql/wip/azure-composition.ytt.yml | 161 +++++++++++++++ .../crossplane/psql/ytt/crossplane.ytt.yml | 13 ++ .../crossplane/psql/ytt/definition.ytt.yml | 88 ++++++++ .../psql/ytt/helm-composition.ytt.yml | 188 ++++++++++++++++++ multicloud/crossplane/psql/ytt/schema.ytt.yml | 42 ++++ 9 files changed, 651 insertions(+) create mode 100644 multicloud/crossplane/psql/README.md create mode 100644 multicloud/crossplane/psql/claim-examples/helm-psql-12.yaml create mode 100644 multicloud/crossplane/psql/claim/helm.yml create mode 100644 multicloud/crossplane/psql/wip/aws-composition.ytt.yml create mode 100644 multicloud/crossplane/psql/wip/azure-composition.ytt.yml create mode 100644 multicloud/crossplane/psql/ytt/crossplane.ytt.yml create mode 100644 multicloud/crossplane/psql/ytt/definition.ytt.yml create mode 100644 multicloud/crossplane/psql/ytt/helm-composition.ytt.yml create mode 100644 multicloud/crossplane/psql/ytt/schema.ytt.yml diff --git a/multicloud/crossplane/psql/README.md b/multicloud/crossplane/psql/README.md new file mode 100644 index 0000000..fc0b6be --- /dev/null +++ b/multicloud/crossplane/psql/README.md @@ -0,0 +1,14 @@ +# Multicloud PSQL - Crossplane + +## Azure + +### Pre-requisites + +* Kubernetes cluster +* Azure Connection Details +* SecretGen Controller + + +## Local (Kubernetes) + +## AWS? diff --git a/multicloud/crossplane/psql/claim-examples/helm-psql-12.yaml b/multicloud/crossplane/psql/claim-examples/helm-psql-12.yaml new file mode 100644 index 0000000..4270056 --- /dev/null +++ b/multicloud/crossplane/psql/claim-examples/helm-psql-12.yaml @@ -0,0 +1,16 @@ +apiVersion: multi.ref.services.apps.tanzu.vmware.com/v1alpha1 +kind: PostgreSQLInstance +metadata: + name: postgresql-049 + labels: + services.apps.tanzu.vmware.com/claimable: "true" +spec: + compositionSelector: + matchLabels: + provider: helm + parameters: + location: local + version: "12" + database: petclinic + collation: en_GB.utf8 + storageClass: gp2 diff --git a/multicloud/crossplane/psql/claim/helm.yml b/multicloud/crossplane/psql/claim/helm.yml new file mode 100644 index 0000000..b1caef7 --- /dev/null +++ b/multicloud/crossplane/psql/claim/helm.yml @@ -0,0 +1,18 @@ +#@ load("@ytt:data", "data") +--- +apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version +kind: #@ data.values.xrd.claimNames.kind +metadata: + name: postgresql-049 + labels: + services.apps.tanzu.vmware.com/claimable: "true" +spec: + compositionSelector: + matchLabels: + provider: helm + parameters: + location: local + version: #@ data.values.version + database: petclinic + collation: en_GB.utf8 + storageClass: gp2 diff --git a/multicloud/crossplane/psql/wip/aws-composition.ytt.yml b/multicloud/crossplane/psql/wip/aws-composition.ytt.yml new file mode 100644 index 0000000..def88fb --- /dev/null +++ b/multicloud/crossplane/psql/wip/aws-composition.ytt.yml @@ -0,0 +1,111 @@ +#@ load("@ytt:data", "data") +--- +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: #@ (data.values.xrd.claimNames.kind).lower() + labels: + crossplane.io/xrd: #@ data.values.xrd.names.plural + "." + data.values.xrd.group + provider: #@ data.values.provider.name + database: #@ data.values.cloudServiceBindingType +spec: + publishConnectionDetailsWithStoreConfigRef: + name: #@ data.values.storeConfig.name + compositeTypeRef: + apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version + kind: #@ data.values.xrd.names.kind + resources: + - base: + apiVersion: rds.aws.upbound.io/v1beta1 + kind: Instance + metadata: + annotations: + upjet.upbound.io/manual-intervention: This resource has a password secret reference. + name: example-dbinstance + spec: + forProvider: + allocatedStorage: 20 + autoMinorVersionUpgrade: true + backupRetentionPeriod: 14 + backupWindow: 09:46-10:16 + engine: postgres + engineVersion: "13.7" + instanceClass: db.t3.micro + maintenanceWindow: Mon:00:00-Mon:03:00 + name: example + passwordSecretRef: #! needs password generated by secret gen as option?! + key: password + name: example-dbinstance + namespace: upbound-system + publiclyAccessible: false + region: us-west-1 + skipFinalSnapshot: true + storageEncrypted: false + storageType: gp2 #! One of "standard" (magnetic), "gp2" (general purpose SSD), or "io1" (provisioned IOPS SSD). The default is "io1" if iops is specified, "gp2" if not. + username: adminuser + optionGroupName: abc #! needs reference/api resouce + parameterGroupName: abc #! needs reference/api resouce + vpcSecurityGroupIdSelector: #! needs reference/api resouce + matchControllerRef: true #! needs reference/api resouce + dbSubnetGroupNameSelector: + matchControllerRef: true #! needs reference/api resouce + providerConfigRef: + name: #@ data.values.provider.configRef + writeConnectionSecretToRef: + namespace: #@ data.values.crossplane.namespace + #! if specified, else it goes into "default VPC" + - base: + apiVersion: rds.aws.upbound.io/v1beta1 + kind: SubnetGroup + metadata: + name: example-subnetgroup + spec: + forProvider: + region: us-west-1 + subnetIdRefs: + - name: sample-db-subnet1 + - name: sample-db-subnet2 + tags: + Name: My DB subnet group + + #! OLD, needs to be replaced + - base: + apiVersion: database.aws.crossplane.io/v1beta1 + kind: RDSInstance + spec: + forProvider: + dbInstanceClass: db.t2.micro + engine: postgres + dbName: postgres + engineVersion: "12" + masterUsername: masteruser + publiclyAccessible: true + region: us-east-1 + skipFinalSnapshotBeforeDeletion: true + writeConnectionSecretToRef: + namespace: #@ data.values.crossplane.namespace + connectionDetails: + - name: type + value: postgresql + - name: provider + value: aws + - name: database + value: postgres + - fromConnectionSecretKey: username + - fromConnectionSecretKey: password + - name: host + fromConnectionSecretKey: endpoint + - fromConnectionSecretKey: port + name: rdsinstance + patches: + - fromFieldPath: metadata.uid + toFieldPath: spec.writeConnectionSecretToRef.name + transforms: + - string: + fmt: '%s-postgresql' + type: Format + type: string + type: FromCompositeFieldPath + - fromFieldPath: spec.parameters.storageGB + toFieldPath: spec.forProvider.allocatedStorage + type: FromCompositeFieldPath \ No newline at end of file diff --git a/multicloud/crossplane/psql/wip/azure-composition.ytt.yml b/multicloud/crossplane/psql/wip/azure-composition.ytt.yml new file mode 100644 index 0000000..701bbad --- /dev/null +++ b/multicloud/crossplane/psql/wip/azure-composition.ytt.yml @@ -0,0 +1,161 @@ +#@ load("@ytt:data", "data") +--- +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: #@ (data.values.xrd.claimNames.kind).lower() + labels: + crossplane.io/xrd: #@ data.values.xrd.names.plural + "." + data.values.xrd.group + provider: #@ data.values.provider.name + database: #@ data.values.cloudServiceBindingType +spec: + publishConnectionDetailsWithStoreConfigRef: + name: #@ data.values.storeConfig.name + compositeTypeRef: + apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version + kind: #@ data.values.xrd.names.kind + resources: + - name: resourcegroup + base: + apiVersion: azure.upbound.io/v1beta1 + kind: ResourceGroup + spec: {} + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.location + - type: ToCompositeFieldPath + fromFieldPath: metadata.name + toFieldPath: status.resourceGroup + - name: flexibleserver + base: + apiVersion: dbforpostgresql.azure.upbound.io/v1beta1 + kind: FlexibleServer + spec: + providerConfigRef: + name: #@ data.values.provider.configRef + forProvider: + resourceGroupNameSelector: + matchControllerRef: true + administratorLogin: psqladmin + administratorPasswordSecretRef: + key: password + name: "" + namespace: #@ data.values.crossplane.namespace + location: West Europe + skuName: GP_Standard_D4s_v3 + storageMb: 32768 + version: "12" #! 11,12 and 13 are supported + connectionDetails: + - name: type + value: postgresql + - name: provider + value: azure + - name: database + value: postgres + - name: username + fromFieldPath: spec.forProvider.administratorLogin + - name: password + fromConnectionSecretKey: "attribute.administrator_password" + - name: host + fromFieldPath: status.atProvider.fqdn + - name: port + type: FromValue + value: "5432" + patches: + - fromFieldPath: metadata.uid + toFieldPath: spec.writeConnectionSecretToRef.name + transforms: + - string: + fmt: '%s-postgresql' + type: Format + type: string + type: FromCompositeFieldPath + - type: FromCompositeFieldPath + fromFieldPath: metadata.name + toFieldPath: spec.forProvider.administratorPasswordSecretRef.name + - name: flexibleserverconfig + base: + apiVersion: dbforpostgresql.azure.upbound.io/v1beta1 + kind: FlexibleServerConfiguration + spec: + providerConfigRef: + name: #@ data.values.provider.configRef + forProvider: + serverIdSelector: + matchControllerRef: true + name: backslash_quote + value: on + - name: flexibleserverdatabase + base: + apiVersion: dbforpostgresql.azure.upbound.io/v1beta1 + kind: FlexibleServerDatabase + spec: + providerConfigRef: + name: #@ data.values.provider.configRef + forProvider: + serverIdSelector: + matchControllerRef: true + charset: "" + collation: "" + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.charset + toFieldPath: spec.forProvider.charset + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.collation + toFieldPath: spec.forProvider.collation + - name: firewallrule + base: + apiVersion: dbforpostgresql.azure.upbound.io/v1beta1 + kind: FlexibleServerFirewallRule + spec: + providerConfigRef: + name: #@ data.values.provider.configRef + forProvider: + serverIdSelector: + matchControllerRef: true + startIpAddress: "" + endIpAddress: "" + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.firewallRule.startIpAddress + toFieldPath: spec.forProvider.startIpAddress + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.firewallRule.endIpAddress + toFieldPath: spec.forProvider.endIpAddress + - name: password + base: + apiVersion: kubernetes.crossplane.io/v1alpha1 + kind: Object + spec: + forProvider: + manifest: + apiVersion: secretgen.k14s.io/v1alpha1 + kind: Password + metadata: + name: "" + namespace: #@ data.values.crossplane.namespace + spec: + length: 64 + secretTemplate: + type: Opaque + stringData: + password: $(value) + patches: + - type: FromCompositeFieldPath + fromFieldPath: metadata.name + toFieldPath: spec.forProvider.manifest.metadata.name + + +#! location: "" +#! server: +#! version: "13" +#! instanceKind: [default, high-performance] + #! instance_type: "Standard_D2s_v3" + #! instance_tier: "GeneralPurpose" +#! storageSize: [small, medium, large] (64, 256, 1024) + #! instance_storage_size_gb: 128 +#! firewall_rules: +#! startIpAddress: "" +#! endIpAddress: "" \ No newline at end of file diff --git a/multicloud/crossplane/psql/ytt/crossplane.ytt.yml b/multicloud/crossplane/psql/ytt/crossplane.ytt.yml new file mode 100644 index 0000000..7f88c1c --- /dev/null +++ b/multicloud/crossplane/psql/ytt/crossplane.ytt.yml @@ -0,0 +1,13 @@ +#@ load("@ytt:data", "data") +--- +apiVersion: meta.pkg.crossplane.io/v1 +kind: Configuration +metadata: + name: #@ data.values.configurationName +spec: + dependsOn: + #@ for name in data.values.providers: + #@ provider = data.values.providers[name] + - provider: #@ provider.image + version: #@ provider.version + #@ end diff --git a/multicloud/crossplane/psql/ytt/definition.ytt.yml b/multicloud/crossplane/psql/ytt/definition.ytt.yml new file mode 100644 index 0000000..731af76 --- /dev/null +++ b/multicloud/crossplane/psql/ytt/definition.ytt.yml @@ -0,0 +1,88 @@ +#@ load("@ytt:data", "data") +--- +apiVersion: apiextensions.crossplane.io/v1 +kind: CompositeResourceDefinition +metadata: + name: #@ data.values.xrd.names.plural + "." + data.values.xrd.group +spec: + group: #@ data.values.xrd.group + names: #@ data.values.xrd.names + claimNames: #@ data.values.xrd.claimNames + connectionSecretKeys: + - type + - provider + - host + - port + - database + - username + - password + versions: + + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + parameters: + type: object + properties: + location: + type: string + #! providerConfig: + #! type: string + #! default: default + version: + type: string + default: "13" + enum: ["11", "12", "13"] + adminUsername: + type: string + default: postgres + storageClass: + type: string + default: default + database: + type: string + default: postgres + collation: + type: string + default: en_US.utf8 + charset: + type: string + default: utf8 + required: + - location + required: + - parameters + status: + type: object + properties: + version: + type: string + address: + type: string + location: + type: string + binding: + type: object + properties: + name: + type: string + additionalPrinterColumns: + - name: address + type: string + jsonPath: .status.address + - name: location + type: string + jsonPath: .status.location + - name: version + type: string + jsonPath: .status.version + - name: connection-details + type: string + jsonPath: .status.binding.name \ No newline at end of file diff --git a/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml b/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml new file mode 100644 index 0000000..d220161 --- /dev/null +++ b/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml @@ -0,0 +1,188 @@ +#@ load("@ytt:data", "data") +--- +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: #@ data.values.providers.helm.name + "-" + data.values.cloudServiceBindingType + labels: + crossplane.io/xrd: #@ data.values.xrd.names.plural + "." + data.values.xrd.group + provider: #@ data.values.providers.helm.name + database: #@ data.values.cloudServiceBindingType +spec: + writeConnectionSecretsToNamespace: #@ data.values.crossplane.namespace + compositeTypeRef: + apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version + kind: #@ data.values.xrd.names.kind + resources: + + - name: tf-providerconfig + base: + apiVersion: kubernetes.crossplane.io/v1alpha1 + kind: Object + spec: + forProvider: + manifest: + apiVersion: tf.crossplane.io/v1alpha1 + kind: ProviderConfig + spec: + credentials: [] + #@yaml/text-templated-strings + configuration: | + terraform { + required_providers { + random = { + source = "hashicorp/random" + version = "3.4.3" + } + } + + backend "kubernetes" { + secret_suffix = "providerconfig-default" + namespace = "(@= data.values.crossplane.namespace @)" + in_cluster_config = true + } + } + + provider "random" {} + + patches: + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.forProvider.manifest.metadata.name + + - name: password + base: + apiVersion: tf.crossplane.io/v1alpha1 + kind: Workspace + spec: + forProvider: + module: | + resource "random_password" "password" { + length = 64 + special = false + } + + output "password" { + value = random_password.password.result + sensitive = true + } + source: Inline + writeConnectionSecretToRef: + namespace: #@ data.values.crossplane.namespace + + patches: + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.writeConnectionSecretToRef.name + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.providerConfigRef.name + - type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-name] + toFieldPath: spec.writeConnectionSecretToRef.name + - type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-namespace] + toFieldPath: spec.writeConnectionSecretToRef.namespace + - type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-name] + toFieldPath: status.binding.name + + connectionDetails: + - fromConnectionSecretKey: password + + - name: release + base: + apiVersion: helm.crossplane.io/v1beta1 + kind: Release + spec: + forProvider: + namespace: #@ data.values.crossplane.namespace + chart: + name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 12.1.2 + values: + architecture: standalone + global: + postgresql: + auth: + secretKeys: + adminPasswordKey: password + + connectionDetails: + - name: type + value: postgresql + - name: provider + value: #@ data.values.providers.helm.name + - name: database + fromFieldPath: spec.forProvider.values.global.postgresql.auth.database + type: FromFieldPath + - name: username + fromFieldPath: spec.forProvider.values.global.postgresql.auth.username + type: FromFieldPath + - name: host + fromFieldPath: metadata.labels.address + type: FromFieldPath + - name: port + value: "5432" + + patches: + - type: CombineFromComposite + toFieldPath: spec.forProvider.values.primary.initdb.args + policy: + fromFieldPath: Required + combine: + strategy: string + string: + fmt: '-E %s --locale=%s' + variables: + - fromFieldPath: spec.parameters.charset + - fromFieldPath: spec.parameters.collation + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.version + toFieldPath: spec.forProvider.values.image.tag + transforms: + - type: map + map: + "11": "11.18.0" + "12": "12.13.0" + "13": "13.9.0" + "14": "14.6.0" + "15": "15.1.0" + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.forProvider.values.global.postgresql.auth.existingSecret + transforms: + - type: string + string: + type: Format + fmt: '%s-postgresql-admin' + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.adminUsername + toFieldPath: spec.forProvider.values.global.postgresql.auth.username + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.storageClass + toFieldPath: spec.forProvider.values.primary.persistence.storageClass + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.database + toFieldPath: spec.forProvider.values.global.postgresql.auth.database + - type: FromCompositeFieldPath + fromFieldPath: status.address + toFieldPath: metadata.labels.address + - type: ToCompositeFieldPath + toFieldPath: status.location + fromFieldPath: spec.forProvider.namespace + - type: CombineToComposite + toFieldPath: status.address + policy: + fromFieldPath: Required + combine: + strategy: string + string: + fmt: '%s.%s.svc.cluster.local' + variables: + - fromFieldPath: metadata.name + - fromFieldPath: spec.forProvider.namespace + - type: ToCompositeFieldPath + fromFieldPath: spec.forProvider.values.image.tag + toFieldPath: status.version \ No newline at end of file diff --git a/multicloud/crossplane/psql/ytt/schema.ytt.yml b/multicloud/crossplane/psql/ytt/schema.ytt.yml new file mode 100644 index 0000000..1df865f --- /dev/null +++ b/multicloud/crossplane/psql/ytt/schema.ytt.yml @@ -0,0 +1,42 @@ +#@data/values-schema +--- + +xrd: + group: multi.ref.services.apps.tanzu.vmware.com + names: + kind: XPostgreSQLInstance + plural: xpostgresqlinstances + claimNames: + kind: PostgreSQLInstance + plural: postgresqlinstances + version: v1alpha1 + +configurationName: "multicloud-psql" +cloudServiceBindingType: "postgresql" + +providers: +#! azure: +#! name: azure +#! image: xpkg.upbound.io/upbound/provider-azure +#! version: ">=v0.18.1" +#! configRef: default + + helm: + name: helm + image: xpkg.upbound.io/crossplane-contrib/provider-helm + version: ">=v0.12.0" + +crossplane: + #@schema/title "CrossplaneNamespace" + #@schema/desc "The namespace where crossplane controller is installed" + namespace: upbound-system + version: '^v1.10' + +#@schema/title "StoreConfig" +#@schema/desc "Details of the StoreConfig" +storeConfig: + #@schema/title "StoreConfig Name" + #@schema/desc "The name of the StoreConfig" + name: "default" + +version: "" From 08775cc3f7247a5e1d0c1d2827db65538e0a8c19 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Wed, 30 Nov 2022 00:36:16 +0100 Subject: [PATCH 02/41] refactor publish and test workloads --- .../azure-crossplane-mongodb-e2e.yml | 81 ---------------- .github/workflows/crossplane-main.yml | 42 -------- .github/workflows/crossplane-prepublish.yml | 43 -------- .github/workflows/publish-packages.yml | 58 +++++++++++ .github/workflows/reusable-bump-version.yml | 35 +++++++ .github/workflows/reusable-carvel-publish.yml | 65 +++++++++++++ .github/workflows/reusable-carvel-test.yml | 77 +++++++++++++++ ...sh.yml => reusable-crossplane-publish.yml} | 44 ++++++--- .../workflows/reusable-crossplane-test.yml | 83 ++++++++++++++++ .github/workflows/reusable-list-packages.yml | 97 +++++++++++++++++++ Makefile | 11 +-- ...crossplane-azure-provider-create-secret.sh | 9 +- scripts/crossplane-e2e-mongodb.sh | 23 +++-- .../claim-instance.sh} | 9 +- .../cleanup.sh} | 2 + .../install-package.sh} | 7 +- .../test.sh} | 2 + scripts/crossplane-install-uxp.sh | 6 +- scripts/kind-cluster.sh | 4 +- 19 files changed, 496 insertions(+), 202 deletions(-) delete mode 100644 .github/workflows/azure-crossplane-mongodb-e2e.yml delete mode 100644 .github/workflows/crossplane-main.yml delete mode 100644 .github/workflows/crossplane-prepublish.yml create mode 100644 .github/workflows/publish-packages.yml create mode 100644 .github/workflows/reusable-bump-version.yml create mode 100644 .github/workflows/reusable-carvel-publish.yml create mode 100644 .github/workflows/reusable-carvel-test.yml rename .github/workflows/{crossplane-publish.yml => reusable-crossplane-publish.yml} (56%) create mode 100644 .github/workflows/reusable-crossplane-test.yml create mode 100644 .github/workflows/reusable-list-packages.yml rename scripts/{mongodb/crossplane-claim-instance.sh => crossplane-e2e-mongodb/claim-instance.sh} (84%) rename scripts/{mongodb/crossplane-e2e-cleanup.sh => crossplane-e2e-mongodb/cleanup.sh} (97%) rename scripts/{mongodb/crossplane-install-package.sh => crossplane-e2e-mongodb/install-package.sh} (86%) rename scripts/{mongodb/crossplane-test.sh => crossplane-e2e-mongodb/test.sh} (97%) diff --git a/.github/workflows/azure-crossplane-mongodb-e2e.yml b/.github/workflows/azure-crossplane-mongodb-e2e.yml deleted file mode 100644 index bf97168..0000000 --- a/.github/workflows/azure-crossplane-mongodb-e2e.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: MongoDB E2E Crossplane - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - pull_request: - types: [opened, ready_for_review, reopened, synchronize] - branches: main - paths: - - 'azure/crossplane/mongodb/**' - -env: - PACKAGE_PROVIDER: azure - PACKAGE_NAME: mongodb - -jobs: - pre-publish: - uses: ./.github/workflows/crossplane-prepublish.yml - - publish: - needs: - - pre-publish - uses: ./.github/workflows/crossplane-publish.yml - with: - package: "mongodb" - provider: "azure" - version: ${{ needs.pre-publish.outputs.version }} - - test: - runs-on: ubuntu-latest - needs: - - pre-publish - - publish - env: - CONFIG_VERSION: ${{ needs.pre-publish.outputs.version }} - CONFIG_IMAGE: ${{ needs.publish.outputs.package_registry }}/${{ needs.publish.outputs.package_repository }} - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Install UXP - run: | - curl -sL "https://cli.upbound.io" | sh - sudo mv up /usr/local/bin/ - up --version - - - name: Create k8s Kind Cluster - uses: helm/kind-action@v1.4.0 - with: - verbosity: 5 - version: "v0.16.0" - kubectl_version: "v1.24.6" - node_image: kindest/node:v1.24.6@sha256:97e8d00bc37a7598a0b32d1fabd155a96355c49fa0d4d4790aab0f161bf31be1 - - - name: Verify Cluster - run: | - kubectl version - which kubectl - kubectl cluster-info - kubectl get storageclass standard - - - name: Create Azure Config Secret - run: | - kubectl create namespace upbound-system - kubectl create secret generic azure-secret -n upbound-system --from-literal=creds='${{ secrets.AZURE_CONFIG }}' - - - name: Test Crossplane MongoDB Package - run: | - pushd scripts - ./crossplane-e2e-mongodb.sh - popd - - - name: Cleanup MongoDB Package - if: always() - run: | - pushd scripts - ./mongodb/crossplane-e2e-cleanup.sh - popd diff --git a/.github/workflows/crossplane-main.yml b/.github/workflows/crossplane-main.yml deleted file mode 100644 index 8970ea4..0000000 --- a/.github/workflows/crossplane-main.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Publish workflow - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - -env: - PACKAGE_REGISTRY: ghcr.io - PACKAGE_REPOSITORY_BASE: ${{ github.repository }} - -on: - push: - branches: - - main - paths: - - 'amazon/crossplane/**' - - 'azure/crossplane/**' - - 'google/crossplane/**' - -jobs: - pre-publish: - uses: ./.github/workflows/crossplane-prepublish.yml - - debug: - runs-on: ubuntu-latest - needs: - - pre-publish - steps: - - run: - echo "packages='${{ needs.pre-publish.outputs.packages }}'" - echo "packages_fromJson='${{ fromJson(needs.pre-publish.outputs.packages) }}'" - - publish: - needs: - - pre-publish - - debug - uses: ./.github/workflows/crossplane-publish.yml - strategy: - matrix: ${{ fromJson(needs.pre-publish.outputs.packages) }} - with: - package: ${{ matrix.PACKAGE_NAME }} - provider: ${{ matrix.PACKAGE_PROVIDER }} - version: ${{ needs.pre-publish.outputs.version }} diff --git a/.github/workflows/crossplane-prepublish.yml b/.github/workflows/crossplane-prepublish.yml deleted file mode 100644 index 0539721..0000000 --- a/.github/workflows/crossplane-prepublish.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Reusable pre-publish workflow - -on: - - workflow_call: - outputs: - version: - value: ${{ jobs.pre-publish.outputs.version }} - description: The new published version - packages: - value: ${{ jobs.pre-publish.outputs.packages }} - -jobs: - pre-publish: - runs-on: ubuntu-latest - outputs: - packages: ${{ steps.list.outputs.packages }} - version: ${{ steps.version_bump.outputs.new_tag }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: '0' - - - name: Bump version and push tag - id: version_bump - uses: anothrNick/github-tag-action@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WITH_V: false - INITIAL_VERSION: 0.1.0 - RELEASE_BRANCHES: main - PRERELEASE: true - DEFAULT_BUMP: patch - - - name: List Packages That Have Changed - id: list - run: | - diffs=$(git diff-tree --no-commit-id --name-only -r --diff-filter=ACM HEAD~1 HEAD | grep -E '^[^/]+/crossplane/' | cut -d/ -f-3 | jq -ncrR '[inputs]|unique') - echo "diffs=${diffs}" - packages=$(jq -nc --argjson diffs "${diffs}" '$diffs|map(split("/")|{"PACKAGE_PROVIDER":.[0],"PACKAGE_NAME":.[2]}) | { "include": . }') - echo "packages=${packages}" - echo "packages=${packages}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml new file mode 100644 index 0000000..2e4674f --- /dev/null +++ b/.github/workflows/publish-packages.yml @@ -0,0 +1,58 @@ +name: Publish packages + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - main + paths: + - 'packages/*/*/*/**' + + pull_request: + types: [opened, ready_for_review, reopened, synchronize] + branches: + - main + paths: + - 'packages/*/*/*/**' + +jobs: + + list-packages: + uses: ./.github/workflows/reusable-list-packages.yml + + bump-version: + needs: + - list-packages + if: needs.list-packages.outputs.crossplane_publish == 'true' || needs.list-packages.outputs.carvel_publish == 'true' + uses: ./.github/workflows/reusable-bump-version.yml + + crossplane-publish: + needs: + - bump-version + - list-packages + if: needs.list-packages.outputs.crossplane_publish == 'true' + strategy: + matrix: ${{ fromJson(needs.list-packages.outputs.crossplane) }} + uses: ./.github/workflows/reusable-crossplane-publish.yml + with: + package_name: ${{ matrix.name }} + package_provider: ${{ matrix.provider }} + package_version: ${{ needs.bump-version.outputs.version }} + run-test: ${{ github.event_name == 'pull_request' }} + + carvel-publish: + needs: + - bump-version + - list-packages + if: needs.list-packages.outputs.carvel_publish == 'true' + strategy: + matrix: ${{ fromJson(needs.list-packages.outputs.carvel) }} + uses: ./.github/workflows/reusable-carvel-publish.yml + with: + package_name: ${{ matrix.name }} + package_provider: ${{ matrix.provider }} + package_version: ${{ needs.bump-version.outputs.version }} + run-test: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/reusable-bump-version.yml b/.github/workflows/reusable-bump-version.yml new file mode 100644 index 0000000..fd848bd --- /dev/null +++ b/.github/workflows/reusable-bump-version.yml @@ -0,0 +1,35 @@ +name: Reusable bump-version workflow + +on: + + workflow_call: + + outputs: + version: + value: ${{ jobs.bump-version.outputs.version }} + description: The new published version + +jobs: + + bump-version: + + runs-on: ubuntu-latest + + outputs: + version: ${{ steps.bump.outputs.new_tag }} + + steps: + + - name: Checkout + uses: actions/checkout@v3 + + - name: Bump version and push tag + id: bump + uses: anothrNick/github-tag-action@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WITH_V: false + INITIAL_VERSION: 0.1.0 + RELEASE_BRANCHES: main + PRERELEASE: true + DEFAULT_BUMP: patch diff --git a/.github/workflows/reusable-carvel-publish.yml b/.github/workflows/reusable-carvel-publish.yml new file mode 100644 index 0000000..445fce5 --- /dev/null +++ b/.github/workflows/reusable-carvel-publish.yml @@ -0,0 +1,65 @@ +name: Reusable workflow for publishing Carvel packages + +on: + + workflow_call: + + inputs: + package_version: + type: string + required: true + package_name: + type: string + required: true + package_provider: + type: string + required: true + run-test: + type: boolean + default: false + kubernetes-version: + type: string + default: v1.24.6 + kind-version: + type: string + default: v0.16.0 + + outputs: + package_repository: + value: ${{ jobs.publish.outputs.package_repository }} + package_registry: + value: ${{ jobs.publish.outputs.package_registry }} + +jobs: + + publish: + runs-on: ubuntu-latest + + outputs: + package_repository: ${{ env.PACKAGE_REPOSITORY }} + package_registry: ${{ env.PACKAGE_REGISTRY }} + + env: + PACKAGE_VERSION: ${{ inputs.package_version }} + PACKAGE_REGISTRY: ghcr.io + PACKAGE_REPOSITORY: ${{ github.repository }}/${{ inputs.package_provider }}/carvel/${{ inputs.package_name }} + PACKAGE_PROVIDER: ${{ inputs.package_provider }} + PACKAGE_NAME: ${{ inputs.package_name }} + + steps: + + - name: Dump info + run: | + env | sort + + test: + if: inputs.run-test + uses: ./.github/workflows/reusable-carvel-test.yml + needs: + - publish + with: + package_version: ${{ inputs.package_version }} + package_name: ${{ inputs.package_name }} + package_provider: ${{ inputs.package_provider }} + package_registry: ${{ needs.publish.outputs.package_registry }} + package_repository: ${{ needs.publish.outputs.package_repository }} diff --git a/.github/workflows/reusable-carvel-test.yml b/.github/workflows/reusable-carvel-test.yml new file mode 100644 index 0000000..173ff04 --- /dev/null +++ b/.github/workflows/reusable-carvel-test.yml @@ -0,0 +1,77 @@ +name: Reusable workflow for testing Carvel packages + +on: + + workflow_call: + + inputs: + package_version: + type: string + required: true + package_name: + type: string + required: true + package_provider: + type: string + required: true + package_registry: + type: string + required: true + package_repository: + type: string + required: true + kubernetes-version: + type: string + default: v1.24.6 + kind-version: + type: string + default: v0.16.0 + +jobs: + + test: + runs-on: ubuntu-latest + env: + CROSSPLANE_NAMESPACE: upbound-system + CONFIG_VERSION: ${{ inputs.package_version }} + CONFIG_IMAGE: ${{ inputs.package_registry }}/${{ inputs.package_repository }} + + steps: + + - name: Dump info + run: | + env | sort + + - name: Create k8s Kind Cluster + uses: helm/kind-action@v1.4.0 + with: + verbosity: 5 + version: ${{ inputs.kind-version }} + kubectl_version: ${{ inputs.kubernetes-version }} + node_image: kindest/node:${{ inputs.kubernetes-version }} + + - name: Verify Cluster + run: | + kubectl version + which kubectl + kubectl cluster-info + kubectl get storageclass standard + + - name: Create Azure Config Secret + if: inputs.package_provider == 'azure' + run: | + kubectl create namespace ${CROSSPLANE_NAMESPACE} + kubectl create secret generic azure-secret -n ${CROSSPLANE_NAMESPACE} --from-literal=creds='${{ secrets.AZURE_CONFIG }}' + + - name: Test Carvel Package + run: | + [ -x ${SCRIPT} ] && ${SCRIPT} || true + env: + SCRIPT: ./scripts/carvel-e2e-${{ inputs.package_name }}.sh + + - name: Cleanup Carvel Package + if: always() + run: | + [ -x ${SCRIPT} ] && ${SCRIPT} || true + env: + SCRIPT: ./scripts/carvel-e2e-${{ inputs.package_name }}/cleanup.sh diff --git a/.github/workflows/crossplane-publish.yml b/.github/workflows/reusable-crossplane-publish.yml similarity index 56% rename from .github/workflows/crossplane-publish.yml rename to .github/workflows/reusable-crossplane-publish.yml index 28d50f6..a29ea0e 100644 --- a/.github/workflows/crossplane-publish.yml +++ b/.github/workflows/reusable-crossplane-publish.yml @@ -1,18 +1,29 @@ -name: Reusable publish workflow - +name: Reusable workflow for publishing Crossplane packages on: + workflow_call: + inputs: - version: + package_version: type: string required: true - package: + package_name: type: string required: true - provider: + package_provider: type: string required: true + run-test: + type: boolean + default: false + kubernetes-version: + type: string + default: v1.24.6 + kind-version: + type: string + default: v0.16.0 + outputs: package_repository: value: ${{ jobs.publish.outputs.package_repository }} @@ -29,17 +40,17 @@ jobs: package_registry: ${{ env.PACKAGE_REGISTRY }} env: - PACKAGE_VERSION: ${{ inputs.version }} + PACKAGE_VERSION: ${{ inputs.package_version }} PACKAGE_REGISTRY: ghcr.io - PACKAGE_REPOSITORY: ${{ github.repository }}/${{ inputs.provider }}/crossplane/${{ inputs.package }} - PACKAGE_PROVIDER: ${{ inputs.provider }} - PACKAGE_NAME: ${{ inputs.package }} + PACKAGE_REPOSITORY: ${{ github.repository }}/${{ inputs.package_provider }}/crossplane/${{ inputs.package_name }} + PACKAGE_PROVIDER: ${{ inputs.package_provider }} + PACKAGE_NAME: ${{ inputs.package_name }} steps: - name: Checkout uses: actions/checkout@v3 with: - ref: ${{ inputs.version }} + ref: ${{ inputs.package_version }} - name: Install UXP run: | @@ -66,5 +77,16 @@ jobs: - name: Crossplane Package Publish run: | - make crossplane-build make crossplane-push + + test: + if: inputs.run-test + uses: ./.github/workflows/reusable-crossplane-test.yml + needs: + - publish + with: + package_version: ${{ inputs.package_version }} + package_name: ${{ inputs.package_name }} + package_provider: ${{ inputs.package_provider }} + package_registry: ${{ needs.publish.outputs.package_registry }} + package_repository: ${{ needs.publish.outputs.package_repository }} diff --git a/.github/workflows/reusable-crossplane-test.yml b/.github/workflows/reusable-crossplane-test.yml new file mode 100644 index 0000000..798df92 --- /dev/null +++ b/.github/workflows/reusable-crossplane-test.yml @@ -0,0 +1,83 @@ +name: Reusable workflow for testing Crossplane packages + + +on: + + workflow_call: + + inputs: + package_version: + type: string + required: true + package_name: + type: string + required: true + package_provider: + type: string + required: true + package_registry: + type: string + required: true + package_repository: + type: string + required: true + kubernetes-version: + type: string + default: v1.24.6 + kind-version: + type: string + default: v0.16.0 + +jobs: + + test: + runs-on: ubuntu-latest + env: + CROSSPLANE_NAMESPACE: upbound-system + CONFIG_VERSION: ${{ inputs.package_version }} + CONFIG_IMAGE: ${{ inputs.package_registry }}/${{ inputs.package_repository }} + + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - name: Install UXP + run: | + curl -sL "https://cli.upbound.io" | sh + sudo mv up /usr/local/bin/ + up --version + + - name: Create k8s Kind Cluster + uses: helm/kind-action@v1.4.0 + with: + verbosity: 5 + version: ${{ inputs.kind-version }} + kubectl_version: ${{ inputs.kubernetes-version }} + node_image: kindest/node:${{ inputs.kubernetes-version }} + + - name: Verify Cluster + run: | + kubectl version + which kubectl + kubectl cluster-info + kubectl get storageclass standard + + - name: Create Azure Config Secret + if: inputs.package_provider == 'azure' + run: | + kubectl create namespace ${CROSSPLANE_NAMESPACE} + kubectl create secret generic azure-secret -n ${CROSSPLANE_NAMESPACE} --from-literal=creds='${{ secrets.AZURE_CONFIG }}' + + - name: Test Crossplane package + run: | + [ -x ${SCRIPT} ] && ${SCRIPT} || true + env: + SCRIPT: ./scripts/crossplane-e2e-${{ inputs.package_name }}.sh + + - name: Cleanup Crossplane package + if: always() + run: | + [ -x ${SCRIPT} ] && ${SCRIPT} || true + env: + SCRIPT: ./scripts/crossplane-e2e-${{ inputs.package_name }}/cleanup.sh diff --git a/.github/workflows/reusable-list-packages.yml b/.github/workflows/reusable-list-packages.yml new file mode 100644 index 0000000..e3dcd49 --- /dev/null +++ b/.github/workflows/reusable-list-packages.yml @@ -0,0 +1,97 @@ +name: Reusable workflow for listing changed packages + +on: + + workflow_call: + + inputs: + basedir: + type: string + description: Packages base directory path + default: packages + ref: + type: string + description: Git reference to compare HEAD with + default: HEAD~1 + + outputs: + all: + value: ${{ jobs.list-packages.outputs.all }} + crossplane: + value: ${{ jobs.list-packages.outputs.crossplane }} + carvel: + value: ${{ jobs.list-packages.outputs.carvel }} + crossplane_publish: + value: ${{ jobs.list-packages.outputs.crossplane_publish }} + carvel_publish: + value: ${{ jobs.list-packages.outputs.carvel_publish }} + +jobs: + + list-packages: + + runs-on: ubuntu-latest + + outputs: + all: ${{ steps.list.outputs.packages }} + crossplane: ${{ steps.filter.outputs.crossplane }} + carvel: ${{ steps.filter.outputs.carvel }} + crossplane_publish: ${{ steps.filter.outputs.crossplane_publish }} + carvel_publish: ${{ steps.filter.outputs.carvel_publish }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: '0' + + # Prepare a list of changed files between $GIT_REF (default: HEAD~1) and HEAD + # that match a specific pattern: BASEDIR/PROVIDER/PACKAGING/NAME/* + # BASEDIR is set to 'packages' by default + # PROVIDER can be the Cloud name ("aws", "azure", "gcp"), "multicloud" in case the package spans multiple clouds/platforms or free-form string (for example, "helm") + # PACKAGING can currently support either "crossplane" or "carvel" + # NAME is the package's name + + # The `git diff-tree` command outputs a list of file paths starting from the root of the repo, one per line, + # that have been either Added, Copied, Modified or Renamed (ACMR). + # This list is fed to `jq` that transforms it into an actual json array, filters by the entries that start with "BASEDIR/" + # and strip the prefix. + # Each entry is then tokenised by /, morphed into an object with "provider" key set as the first token, "packaging" as the second one and "name" as the third one. + # The resulting array is then purged of the duplicates and then set as value for the "include" key in a new object, + # in order to meet https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#expanding-or-adding-matrix-configurations. + # The result is appended to GITHUB_OUTPUT as well as written to the workflow's output for debugging purposes (thus the `tee -a`). + - name: List packages that have changed + id: list + run: | + packages=$(git diff-tree --name-only -r --diff-filter=ACMR ${GIT_REF} HEAD | jq -ncrR --arg BASEDIR ${BASEDIR%%/} ' + [inputs] + | map(capture("^"+$BASEDIR+"/(?

.*)$").p + | split("/") + | { "provider": .[0], "packaging": .[1], "name": .[2] } + ) + | map(select(.provider!=null and .packaging!=null and .name!=null)) + | unique + | { "include": . } + ') + echo "packages=${packages}" | tee -a $GITHUB_OUTPUT + env: + BASEDIR: ${{ inputs.basedir }} + GIT_REF: ${{ inputs.ref }} + + # Filter packages by packaging system, in order to feed them to different workflows for publishing and testing. + # The $PACKAGES variable is filled with the result from the previous step and filtered using `jq` based on the "packaging" key. + # Currently, only "crossplane" and "carvel" are supported and therefore only those outputs are provided, and contain + # the list of packages per packaging system that have changed (as value of the "include" key as before). + # Boolean outputs are also provided ("crossplane_publish" or "carvel_publish") to tell whether the above list has at least 1 item or not. + - name: Filter packages + id: filter + run: | + crossplane=$(jq -c '.include|=map(select(.packaging=="crossplane"))' <<< $PACKAGES) + echo "crossplane=${crossplane}" | tee -a $GITHUB_OUTPUT + echo "crossplane_publish=$(jq '.include|length > 0' <<<${crossplane})" | tee -a $GITHUB_OUTPUT + + carvel=$(jq -c '.include|=map(select(.packaging=="carvel"))' <<< $PACKAGES) + echo "carvel=${carvel}" | tee -a $GITHUB_OUTPUT + echo "carvel_publish=$(jq '.include|length > 0' <<<${carvel})" | tee -a $GITHUB_OUTPUT + env: + PACKAGES: ${{ steps.list.outputs.packages }} diff --git a/Makefile b/Makefile index c067a89..ed192cf 100644 --- a/Makefile +++ b/Makefile @@ -91,22 +91,17 @@ repo-package-copy: repo-prepare crossplane-ytt: mkdir -p ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/src/ - ytt -f ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/ytt/crossplane.ytt.yml \ - -f ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/ytt/schema.ytt.yml > ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/src/crossplane.yml - ytt -f ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/ytt/definition.ytt.yml \ - -f ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/ytt/schema.ytt.yml > ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/src/definition.yml - ytt -f ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/ytt/composition.ytt.yml \ - -f ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/ytt/schema.ytt.yml > ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/src/composition.yml + ytt -f ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/ytt > ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/src/crossplane.yml crossplane-build: crossplane-ytt - rm ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package/*.xpkg || true mkdir -p ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package + rm ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package/*.xpkg || true up xpkg build \ --package-root ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/src \ --examples-root ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/claim-examples \ --output ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package/crossplane-${PACKAGE_NAME}.xpkg -crossplane-push: +crossplane-push: crossplane-build up xpkg push --package ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package/crossplane-${PACKAGE_NAME}.xpkg \ ${PACKAGE_REGISTRY}/${PACKAGE_REPOSITORY}:${PACKAGE_VERSION} \ --domain ${PACKAGE_REGISTRY} \ No newline at end of file diff --git a/scripts/crossplane-azure-provider-create-secret.sh b/scripts/crossplane-azure-provider-create-secret.sh index e73f6c6..3e16ffb 100755 --- a/scripts/crossplane-azure-provider-create-secret.sh +++ b/scripts/crossplane-azure-provider-create-secret.sh @@ -1,5 +1,10 @@ +#!/usr/bin/env bash + SECRET_NAME=$1 CREDS=$2 +CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} -echo ">> Create Azure Config Secret - secret name=${SECRET_NAME}" -kubectl create secret generic "${SECRET_NAME}" -n upbound-system --from-literal=creds=${CREDS} || true \ No newline at end of file +if [ ! -x ${CREDS} ]; then + echo ">> Create Azure Config Secret - secret name=${SECRET_NAME}" + kubectl create secret generic ${SECRET_NAME} -n ${CROSSPLANE_NAMESPACE} --from-literal=creds=${CREDS} +fi diff --git a/scripts/crossplane-e2e-mongodb.sh b/scripts/crossplane-e2e-mongodb.sh index 49ddc27..927197b 100755 --- a/scripts/crossplane-e2e-mongodb.sh +++ b/scripts/crossplane-e2e-mongodb.sh @@ -1,24 +1,31 @@ +#!/usr/bin/env bash + echo ">> Local Test" +export CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} AZURE_CREDS_SECRET_NAME=${AZURE_CREDS_SECRET_NAME:-"azure-secret"} UXP_VERSION=${UXP_VERSION:-"v1.10.1-up.1"} -export CONFIG_NAME=${CONFIG_NAME:-"trp-azure-mongodb"} +CONFIG_NAME=${CONFIG_NAME:-"trp-azure-mongodb"} CONFIG_IMAGE=${CONFIG_IMAGE:-"ghcr.io/vmware-tanzu-labs/tap-reference-packages-azure/crossplane-mongodb"} CONFIG_VERSION=${CONFIG_VERSION:-"0.23.1-beta.0"} -export CLAIM_NAME=${CLAIM_NAME:-"trp-cosmosdb-mongo-08"} -export TEST_APP_NAME=${TEST_APP_NAME:-"spring-boot-mongo"} +CLAIM_NAME=${CLAIM_NAME:-"trp-cosmosdb-mongo-08"} +TEST_APP_NAME=${TEST_APP_NAME:-"spring-boot-mongo"} + +kubectl create namespace ${CROSSPLANE_NAMESPACE} || true -kubectl create namespace upbound-system || true +pushd $(dirname $0) # Requires AZURE_CONFIG to contain an Azure API credential config (JSON format) ./crossplane-azure-provider-create-secret.sh ${AZURE_CREDS_SECRET_NAME} "${AZURE_CONFIG}" ./crossplane-install-uxp.sh ${UXP_VERSION} -./mongodb/crossplane-install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} ${AZURE_CREDS_SECRET_NAME} +./crossplane-e2e-mongodb/crossplane-install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} ${AZURE_CREDS_SECRET_NAME} + +./crossplane-e2e-mongodb/crossplane-claim-instance.sh ${CLAIM_NAME} -./mongodb/crossplane-claim-instance.sh ${CLAIM_NAME} +./crossplane-e2e-mongodb/crossplane-test.sh ${TEST_APP_NAME} -./mongodb/crossplane-test.sh ${TEST_APP_NAME} +./crossplane-e2e-mongodb/crossplane-test.sh ${TEST_APP_NAME} -./mongodb/crossplane-test.sh ${TEST_APP_NAME} +popd diff --git a/scripts/mongodb/crossplane-claim-instance.sh b/scripts/crossplane-e2e-mongodb/claim-instance.sh similarity index 84% rename from scripts/mongodb/crossplane-claim-instance.sh rename to scripts/crossplane-e2e-mongodb/claim-instance.sh index 22ba1a0..28f25d5 100755 --- a/scripts/mongodb/crossplane-claim-instance.sh +++ b/scripts/crossplane-e2e-mongodb/claim-instance.sh @@ -1,4 +1,7 @@ +#!/usr/bin/env bash + CLAIM_NAME=$1 +CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} echo ">> Claiming a MongoDBInstance" cat <> Showing Secrets (1)" -kubectl get secret -n upbound-system +kubectl get secret -n ${CROSSPLANE_NAMESPACE} kubectl get secret echo ">> Waiting for Managed Resources To Get Ready" kubectl wait --for=condition=ready mongodbinstances.azure.ref.services.apps.tanzu.vmware.com ${CLAIM_NAME} --timeout=400s echo ">> Showing Secrets (2)" -kubectl get secret -n upbound-system +kubectl get secret -n ${CROSSPLANE_NAMESPACE} kubectl get secret echo ">> Showing Comp and Claim status" -kubectl get xmongodbinstance,mongodbinstance \ No newline at end of file +kubectl get xmongodbinstance,mongodbinstance diff --git a/scripts/mongodb/crossplane-e2e-cleanup.sh b/scripts/crossplane-e2e-mongodb/cleanup.sh similarity index 97% rename from scripts/mongodb/crossplane-e2e-cleanup.sh rename to scripts/crossplane-e2e-mongodb/cleanup.sh index 4cc9dd3..c71fe5d 100755 --- a/scripts/mongodb/crossplane-e2e-cleanup.sh +++ b/scripts/crossplane-e2e-mongodb/cleanup.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + CLAIM_NAME=${CLAIM_NAME:-"trp-cosmosdb-mongo-08"} TEST_APP_NAME=${TEST_APP_NAME:-"spring-boot-mongo"} CONFIG_NAME=${CONFIG_NAME:-"trp-azure-mongodb"} diff --git a/scripts/mongodb/crossplane-install-package.sh b/scripts/crossplane-e2e-mongodb/install-package.sh similarity index 86% rename from scripts/mongodb/crossplane-install-package.sh rename to scripts/crossplane-e2e-mongodb/install-package.sh index 23ab550..be41cd1 100755 --- a/scripts/mongodb/crossplane-install-package.sh +++ b/scripts/crossplane-e2e-mongodb/install-package.sh @@ -1,7 +1,10 @@ +#!/usr/bin/env bash + CONFIG_NAME=$1 CONFIG_IMAGE=$2 CONFIG_VERSION=$3 AZURE_CONFIG_SECRET_NAME=$4 +CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} echo ">> Installing Crossplane Package via Configuration CR" cat <> Create Azure Provider Config" cat <> Waiting on Test Application: ${TEST_APP_NAME}" diff --git a/scripts/crossplane-install-uxp.sh b/scripts/crossplane-install-uxp.sh index 34faa64..fb018e8 100755 --- a/scripts/crossplane-install-uxp.sh +++ b/scripts/crossplane-install-uxp.sh @@ -1,4 +1,8 @@ +#!/usr/bin/env bash + UXP_VERSION=${1} +CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} + echo ">> Installing UXP - Universal Crossplane" up uxp install --set 'args={--enable-external-secret-stores}' ${UXP_VERSION} -kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=cloud-infrastructure-controller --namespace upbound-system \ No newline at end of file +kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=cloud-infrastructure-controller --namespace ${CROSSPLANE_NAMESPACE} diff --git a/scripts/kind-cluster.sh b/scripts/kind-cluster.sh index db852ab..7f498fd 100755 --- a/scripts/kind-cluster.sh +++ b/scripts/kind-cluster.sh @@ -1 +1,3 @@ -kind create cluster --name test --wait 5m \ No newline at end of file +#!/usr/bin/env bash + +kind create cluster --name test --wait 5m From eabb607486b056edb3c78d0c764b97f9967a0204 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Thu, 1 Dec 2022 10:15:25 +0100 Subject: [PATCH 03/41] change Makefile to use new packages location --- Makefile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index ed192cf..9d9e056 100644 --- a/Makefile +++ b/Makefile @@ -90,18 +90,18 @@ repo-package-copy: repo-prepare cp -a ${PACKAGES_REPO_STAGING_DIR}/packages/$(shell yq e .metadata.name ${PACKAGE_DIR}/package-metadata.yml)/ ${REPOSITORY_CLOUD_DIR}/${PACKAGE_NAME} crossplane-ytt: - mkdir -p ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/src/ - ytt -f ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/ytt > ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/src/crossplane.yml + mkdir -p ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/src/ + ytt -f ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/ytt > ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/src/crossplane.yml crossplane-build: crossplane-ytt - mkdir -p ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package - rm ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package/*.xpkg || true + mkdir -p ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package + rm ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package/*.xpkg || true up xpkg build \ - --package-root ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/src \ - --examples-root ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/claim-examples \ - --output ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package/crossplane-${PACKAGE_NAME}.xpkg + --package-root ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/src \ + --examples-root ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/claim-examples \ + --output ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package/crossplane-${PACKAGE_NAME}.xpkg crossplane-push: crossplane-build - up xpkg push --package ${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package/crossplane-${PACKAGE_NAME}.xpkg \ + up xpkg push --package ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package/crossplane-${PACKAGE_NAME}.xpkg \ ${PACKAGE_REGISTRY}/${PACKAGE_REPOSITORY}:${PACKAGE_VERSION} \ --domain ${PACKAGE_REGISTRY} \ No newline at end of file From a8e6a141cc60104a1be5f6e0b9a5e8764a43948c Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Thu, 1 Dec 2022 16:36:12 +0100 Subject: [PATCH 04/41] fix publish workflow --- .github/workflows/publish-packages.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index 2e4674f..7cf9c9a 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -20,8 +20,25 @@ on: jobs: + parameters: + runs-on: ubuntu-latest + + outputs: + packages_basedir: ${{ steps.params.outputs.packages_basedir}} + + steps: + + - name: Set parameters + id: params + run: | + echo "packages_basedir=${PACKAGES_BASEDIR}" >> $GITHUB_OUTPUT + list-packages: + needs: + - parameters uses: ./.github/workflows/reusable-list-packages.yml + with: + basedir: ${{ needs.parameters.outputs.packages_basedir }} bump-version: needs: @@ -31,6 +48,7 @@ jobs: crossplane-publish: needs: + - parameters - bump-version - list-packages if: needs.list-packages.outputs.crossplane_publish == 'true' @@ -42,9 +60,11 @@ jobs: package_provider: ${{ matrix.provider }} package_version: ${{ needs.bump-version.outputs.version }} run-test: ${{ github.event_name == 'pull_request' }} + packages_basedir: ${{ needs.parameters.outputs.packages_basedir }} carvel-publish: needs: + - parameters - bump-version - list-packages if: needs.list-packages.outputs.carvel_publish == 'true' @@ -56,3 +76,4 @@ jobs: package_provider: ${{ matrix.provider }} package_version: ${{ needs.bump-version.outputs.version }} run-test: ${{ github.event_name == 'pull_request' }} + packages_basedir: ${{ needs.parameters.outputs.packages_basedir }} From 653ad57fdaa7809a8d4a2105beef7a5a53f256e1 Mon Sep 17 00:00:00 2001 From: Joost van der Griendt Date: Mon, 5 Dec 2022 16:15:46 +0100 Subject: [PATCH 05/41] move and fine tune package --- .../multicloud}/crossplane/psql/README.md | 0 .../crossplane/psql/claim-examples/helm-psql-12.yaml | 0 .../multicloud}/crossplane/psql/claim/helm.yml | 0 .../multicloud}/crossplane/psql/wip/aws-composition.ytt.yml | 0 .../crossplane/psql/wip/azure-composition.ytt.yml | 0 .../multicloud}/crossplane/psql/ytt/crossplane.ytt.yml | 0 .../multicloud}/crossplane/psql/ytt/definition.ytt.yml | 0 .../multicloud}/crossplane/psql/ytt/helm-composition.ytt.yml | 5 +++++ .../multicloud}/crossplane/psql/ytt/schema.ytt.yml | 0 9 files changed, 5 insertions(+) rename {multicloud => packages/multicloud}/crossplane/psql/README.md (100%) rename {multicloud => packages/multicloud}/crossplane/psql/claim-examples/helm-psql-12.yaml (100%) rename {multicloud => packages/multicloud}/crossplane/psql/claim/helm.yml (100%) rename {multicloud => packages/multicloud}/crossplane/psql/wip/aws-composition.ytt.yml (100%) rename {multicloud => packages/multicloud}/crossplane/psql/wip/azure-composition.ytt.yml (100%) rename {multicloud => packages/multicloud}/crossplane/psql/ytt/crossplane.ytt.yml (100%) rename {multicloud => packages/multicloud}/crossplane/psql/ytt/definition.ytt.yml (100%) rename {multicloud => packages/multicloud}/crossplane/psql/ytt/helm-composition.ytt.yml (98%) rename {multicloud => packages/multicloud}/crossplane/psql/ytt/schema.ytt.yml (100%) diff --git a/multicloud/crossplane/psql/README.md b/packages/multicloud/crossplane/psql/README.md similarity index 100% rename from multicloud/crossplane/psql/README.md rename to packages/multicloud/crossplane/psql/README.md diff --git a/multicloud/crossplane/psql/claim-examples/helm-psql-12.yaml b/packages/multicloud/crossplane/psql/claim-examples/helm-psql-12.yaml similarity index 100% rename from multicloud/crossplane/psql/claim-examples/helm-psql-12.yaml rename to packages/multicloud/crossplane/psql/claim-examples/helm-psql-12.yaml diff --git a/multicloud/crossplane/psql/claim/helm.yml b/packages/multicloud/crossplane/psql/claim/helm.yml similarity index 100% rename from multicloud/crossplane/psql/claim/helm.yml rename to packages/multicloud/crossplane/psql/claim/helm.yml diff --git a/multicloud/crossplane/psql/wip/aws-composition.ytt.yml b/packages/multicloud/crossplane/psql/wip/aws-composition.ytt.yml similarity index 100% rename from multicloud/crossplane/psql/wip/aws-composition.ytt.yml rename to packages/multicloud/crossplane/psql/wip/aws-composition.ytt.yml diff --git a/multicloud/crossplane/psql/wip/azure-composition.ytt.yml b/packages/multicloud/crossplane/psql/wip/azure-composition.ytt.yml similarity index 100% rename from multicloud/crossplane/psql/wip/azure-composition.ytt.yml rename to packages/multicloud/crossplane/psql/wip/azure-composition.ytt.yml diff --git a/multicloud/crossplane/psql/ytt/crossplane.ytt.yml b/packages/multicloud/crossplane/psql/ytt/crossplane.ytt.yml similarity index 100% rename from multicloud/crossplane/psql/ytt/crossplane.ytt.yml rename to packages/multicloud/crossplane/psql/ytt/crossplane.ytt.yml diff --git a/multicloud/crossplane/psql/ytt/definition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml similarity index 100% rename from multicloud/crossplane/psql/ytt/definition.ytt.yml rename to packages/multicloud/crossplane/psql/ytt/definition.ytt.yml diff --git a/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml similarity index 98% rename from multicloud/crossplane/psql/ytt/helm-composition.ytt.yml rename to packages/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml index d220161..6b1f234 100644 --- a/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml @@ -74,6 +74,11 @@ spec: - type: FromCompositeFieldPath fromFieldPath: metadata.uid toFieldPath: spec.writeConnectionSecretToRef.name + transforms: + - type: string + string: + type: Format + fmt: '%s-postgresql-admin' - type: FromCompositeFieldPath fromFieldPath: metadata.uid toFieldPath: spec.providerConfigRef.name diff --git a/multicloud/crossplane/psql/ytt/schema.ytt.yml b/packages/multicloud/crossplane/psql/ytt/schema.ytt.yml similarity index 100% rename from multicloud/crossplane/psql/ytt/schema.ytt.yml rename to packages/multicloud/crossplane/psql/ytt/schema.ytt.yml From 1a6736a8a405ac9dc7f927155acdeb6386c88781 Mon Sep 17 00:00:00 2001 From: Joost van der Griendt Date: Mon, 5 Dec 2022 16:38:41 +0100 Subject: [PATCH 06/41] add automation --- .../crossplane-e2e-install-helm-provider.sh | 18 +++++++ .../crossplane-e2e-install-k8s-provider.sh | 18 +++++++ scripts/crossplane-e2e-install-tf-provider.sh | 5 ++ scripts/crossplane-e2e-multicloud-psql.sh | 39 +++++++++++++++ .../claim-helm-instance.sh | 47 +++++++++++++++++++ .../crossplane-e2e-multicloud-psql/cleanup.sh | 19 ++++++++ .../install-package.sh | 26 ++++++++++ .../crossplane-e2e-multicloud-psql/test.sh | 25 ++++++++++ 8 files changed, 197 insertions(+) create mode 100755 scripts/crossplane-e2e-install-helm-provider.sh create mode 100755 scripts/crossplane-e2e-install-k8s-provider.sh create mode 100755 scripts/crossplane-e2e-install-tf-provider.sh create mode 100755 scripts/crossplane-e2e-multicloud-psql.sh create mode 100755 scripts/crossplane-e2e-multicloud-psql/claim-helm-instance.sh create mode 100755 scripts/crossplane-e2e-multicloud-psql/cleanup.sh create mode 100755 scripts/crossplane-e2e-multicloud-psql/install-package.sh create mode 100755 scripts/crossplane-e2e-multicloud-psql/test.sh diff --git a/scripts/crossplane-e2e-install-helm-provider.sh b/scripts/crossplane-e2e-install-helm-provider.sh new file mode 100755 index 0000000..46e27d4 --- /dev/null +++ b/scripts/crossplane-e2e-install-helm-provider.sh @@ -0,0 +1,18 @@ +up controlplane provider install xpkg.upbound.io/crossplane-contrib/provider-helm:v0.12.0 || true + +kubectl wait --for=condition="Healthy" providers.pkg.crossplane.io crossplane-contrib-provider-helm + +sleep 3 + +cat <> Local Test" + +export CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} +#AZURE_CREDS_SECRET_NAME=${AZURE_CREDS_SECRET_NAME:-"azure-secret"} +UXP_VERSION=${UXP_VERSION:-"v1.10.1-up.1"} +CONFIG_NAME=${CONFIG_NAME:-"trp-multicloud-psql"} +CONFIG_IMAGE=${CONFIG_IMAGE:-"ghcr.io/vmware-tanzu-labs/trp-azure-psql"} +CONFIG_VERSION=${CONFIG_VERSION:-"0.0.0-0.0.11-10-geabb607"} +CLAIM_NAME=${CLAIM_NAME:-"postgresql-0001"} +TEST_APP_NAME=${TEST_APP_NAME:-"spring-boot-postgres"} +STORAGE_CLASS=${STORAGE_CLASS:-"default"} + +kubectl create namespace ${CROSSPLANE_NAMESPACE} || true + +pushd $(dirname $0) + +# TODO use this when we also have the Azure version +# Requires AZURE_CONFIG to contain an Azure API credential config (JSON format) +# ./crossplane-azure-provider-create-secret.sh ${AZURE_CREDS_SECRET_NAME} "${AZURE_CONFIG}" + +./crossplane-install-uxp.sh ${UXP_VERSION} + + +echo "> Installing required providers" + +./crossplane-e2e-install-helm-provider.sh +./crossplane-e2e-install-k8s-provider.sh +./crossplane-e2e-install-tf-provider.sh + + +./crossplane-e2e-multicloud-psql/install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} + +./crossplane-e2e-multicloud-psql/claim-helm-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} + +./crossplane-e2e-multicloud-psql/test.sh ${TEST_APP_NAME} + +popd diff --git a/scripts/crossplane-e2e-multicloud-psql/claim-helm-instance.sh b/scripts/crossplane-e2e-multicloud-psql/claim-helm-instance.sh new file mode 100755 index 0000000..ad36506 --- /dev/null +++ b/scripts/crossplane-e2e-multicloud-psql/claim-helm-instance.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +CLAIM_NAME=$1 +STORAGE_CLASS=$2 +CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} + +echo ">> Claiming a PSQl Instance" +cat <> Installing Test Application" +kubectl apply -f https://raw.githubusercontent.com/joostvdg/spring-boot-postgres/main/kubernetes/deployment.yaml +kubectl get deployment + +echo ">> Showing Secrets (1)" +kubectl get secret -n ${CROSSPLANE_NAMESPACE} +kubectl get secret + +echo ">> Waiting for Managed Resources To Get Ready" +kubectl wait --for=condition=ready postgresqlinstances.multi.ref.services.apps.tanzu.vmware.com/${CLAIM_NAME} --timeout=400s +# We can also wait on the "release" + +echo ">> Showing Secrets (2)" +kubectl get secret -n ${CROSSPLANE_NAMESPACE} +kubectl get secret + +echo ">> Showing Comp and Claim status" +kubectl get xpostgresqlinstances,postgresqlinstances diff --git a/scripts/crossplane-e2e-multicloud-psql/cleanup.sh b/scripts/crossplane-e2e-multicloud-psql/cleanup.sh new file mode 100755 index 0000000..1811256 --- /dev/null +++ b/scripts/crossplane-e2e-multicloud-psql/cleanup.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +CLAIM_NAME=${CLAIM_NAME:-"postgresql-0001"} +TEST_APP_NAME=${TEST_APP_NAME:-"spring-boot-postgres"} +CONFIG_NAME=${CONFIG_NAME:-"trp-multicloud-psql"} + +echo ">> Cleaning Up Resources" +echo " > WARNING -> This can take a while (~10 minutes), as it waits for Azure to delete the resources!" + +kubectl delete postgresqlinstances ${CLAIM_NAME} || true +kubectl delete deploy ${TEST_APP_NAME} || true +kubectl delete configuration ${CONFIG_NAME} || true + +kubectl delete providerconfigs.helm.crossplane.io default || true +kubectl delete providerconfigs.kubernetes.crossplane.io default || true + +kubectl delete providers.pkg.crossplane.io crossplane-contrib-provider-helm || true +kubectl delete providers.pkg.crossplane.io crossplane-contrib-provider-kubernetes || true +kubectl delete providers.pkg.crossplane.io crossplane-contrib-provider-terraform || true \ No newline at end of file diff --git a/scripts/crossplane-e2e-multicloud-psql/install-package.sh b/scripts/crossplane-e2e-multicloud-psql/install-package.sh new file mode 100755 index 0000000..dd0ded0 --- /dev/null +++ b/scripts/crossplane-e2e-multicloud-psql/install-package.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +CONFIG_NAME=$1 +CONFIG_IMAGE=$2 +CONFIG_VERSION=$3 +CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} + +echo ">> Installing Crossplane Package via Configuration CR" +cat <> Waiting on Test Application: ${TEST_APP_NAME}" +kubectl get pod +kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=${TEST_APP_NAME} --timeout=60s + +echo ">> Starting Port Forward" +kubectl port-forward deployment/${TEST_APP_NAME} 8080 & +PORT_FORWARD_PID=$! + +sleep 10 + +echo ">> Testing Application" +curl -s "http://localhost:8080" +curl --header "Content-Type: application/json" --request POST --data '{"name":"Piet"}' http://localhost:8080/create +curl --header "Content-Type: application/json" --request POST --data '{"name":"Andrea"}' http://localhost:8080/create +HTTP_RESULT=$(curl -s "http://localhost:8080") +[ $(jq 'map(select(.name =="Piet")) | length'<<<$HTTP_RESULT) -eq 1 ] +[ $(jq 'map(select(.name =="Andrea")) | length'<<<$HTTP_RESULT) -eq 1 ] + +echo ">> Killing Port Forward" +echo " > PORT_FORWARD_PID=$PORT_FORWARD_PID" +kill -9 $PORT_FORWARD_PID \ No newline at end of file From 80c32a246fcc4a25f22a4d0aab6626285bba0365 Mon Sep 17 00:00:00 2001 From: Joost van der Griendt Date: Tue, 6 Dec 2022 11:06:11 +0100 Subject: [PATCH 07/41] initial version of azure composition --- .../{wip => ytt}/azure-composition.ytt.yml | 166 ++++++++++++------ .../crossplane/psql/ytt/definition.ytt.yml | 12 +- .../crossplane/psql/ytt/schema.ytt.yml | 10 +- .../crossplane-e2e-install-azure-provider.sh | 24 +++ scripts/crossplane-e2e-multicloud-psql.sh | 3 +- .../claim-azure-instance.sh | 46 +++++ 6 files changed, 198 insertions(+), 63 deletions(-) rename packages/multicloud/crossplane/psql/{wip => ytt}/azure-composition.ytt.yml (56%) create mode 100755 scripts/crossplane-e2e-install-azure-provider.sh create mode 100755 scripts/crossplane-e2e-multicloud-psql/claim-azure-instance.sh diff --git a/packages/multicloud/crossplane/psql/wip/azure-composition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml similarity index 56% rename from packages/multicloud/crossplane/psql/wip/azure-composition.ytt.yml rename to packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml index 701bbad..d62c228 100644 --- a/packages/multicloud/crossplane/psql/wip/azure-composition.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml @@ -3,18 +3,91 @@ apiVersion: apiextensions.crossplane.io/v1 kind: Composition metadata: - name: #@ (data.values.xrd.claimNames.kind).lower() + name: #@ data.values.providers.azure.name + "-" + data.values.cloudServiceBindingType labels: crossplane.io/xrd: #@ data.values.xrd.names.plural + "." + data.values.xrd.group - provider: #@ data.values.provider.name + provider: #@ data.values.providers.azure.name database: #@ data.values.cloudServiceBindingType spec: - publishConnectionDetailsWithStoreConfigRef: - name: #@ data.values.storeConfig.name + writeConnectionSecretsToNamespace: #@ data.values.crossplane.namespace compositeTypeRef: apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version kind: #@ data.values.xrd.names.kind resources: + - name: tf-providerconfig + base: + apiVersion: kubernetes.crossplane.io/v1alpha1 + kind: Object + spec: + forProvider: + manifest: + apiVersion: tf.crossplane.io/v1alpha1 + kind: ProviderConfig + spec: + credentials: [] + #@yaml/text-templated-strings + configuration: | + terraform { + required_providers { + random = { + source = "hashicorp/random" + version = "3.4.3" + } + } + + backend "kubernetes" { + secret_suffix = "providerconfig-default" + namespace = "(@= data.values.crossplane.namespace @)" + in_cluster_config = true + } + } + provider "random" {} + patches: + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.forProvider.manifest.metadata.name + - name: password + base: + apiVersion: tf.crossplane.io/v1alpha1 + kind: Workspace + spec: + forProvider: + module: | + resource "random_password" "password" { + length = 64 + special = false + } + + output "password" { + value = random_password.password.result + sensitive = true + } + source: Inline + writeConnectionSecretToRef: + namespace: #@ data.values.crossplane.namespace + patches: + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.writeConnectionSecretToRef.name + transforms: + - type: string + string: + type: Format + fmt: '%s-postgresql-admin' + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.providerConfigRef.name + - type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-name] + toFieldPath: spec.writeConnectionSecretToRef.name + - type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-namespace] + toFieldPath: spec.writeConnectionSecretToRef.namespace + - type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-name] + toFieldPath: status.binding.name + connectionDetails: + - fromConnectionSecretKey: password - name: resourcegroup base: apiVersion: azure.upbound.io/v1beta1 @@ -38,14 +111,45 @@ spec: resourceGroupNameSelector: matchControllerRef: true administratorLogin: psqladmin + skuName: GP_Standard_D4s_v3 + storageMb: 32768 administratorPasswordSecretRef: key: password name: "" namespace: #@ data.values.crossplane.namespace - location: West Europe - skuName: GP_Standard_D4s_v3 - storageMb: 32768 - version: "12" #! 11,12 and 13 are supported + location: "" + version: "" #! 11,12 and 13 are supported + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.location + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.version + toFieldPath: spec.forProvider.values.image.tag + transforms: + - type: map + map: + "11": "11" + "12": "12" + "13": "13" + "14": "13" + "15": "13" + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.writeConnectionSecretToRef.name + transforms: + - string: + fmt: '%s-postgresql' + type: Format + type: string + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.forProvider.administratorPasswordSecretRef.name + transforms: + - string: + fmt: '%s-postgresql' + type: Format + type: string connectionDetails: - name: type value: postgresql @@ -62,18 +166,6 @@ spec: - name: port type: FromValue value: "5432" - patches: - - fromFieldPath: metadata.uid - toFieldPath: spec.writeConnectionSecretToRef.name - transforms: - - string: - fmt: '%s-postgresql' - type: Format - type: string - type: FromCompositeFieldPath - - type: FromCompositeFieldPath - fromFieldPath: metadata.name - toFieldPath: spec.forProvider.administratorPasswordSecretRef.name - name: flexibleserverconfig base: apiVersion: dbforpostgresql.azure.upbound.io/v1beta1 @@ -124,38 +216,4 @@ spec: - type: FromCompositeFieldPath fromFieldPath: spec.parameters.firewallRule.endIpAddress toFieldPath: spec.forProvider.endIpAddress - - name: password - base: - apiVersion: kubernetes.crossplane.io/v1alpha1 - kind: Object - spec: - forProvider: - manifest: - apiVersion: secretgen.k14s.io/v1alpha1 - kind: Password - metadata: - name: "" - namespace: #@ data.values.crossplane.namespace - spec: - length: 64 - secretTemplate: - type: Opaque - stringData: - password: $(value) - patches: - - type: FromCompositeFieldPath - fromFieldPath: metadata.name - toFieldPath: spec.forProvider.manifest.metadata.name - -#! location: "" -#! server: -#! version: "13" -#! instanceKind: [default, high-performance] - #! instance_type: "Standard_D2s_v3" - #! instance_tier: "GeneralPurpose" -#! storageSize: [small, medium, large] (64, 256, 1024) - #! instance_storage_size_gb: 128 -#! firewall_rules: -#! startIpAddress: "" -#! endIpAddress: "" \ No newline at end of file diff --git a/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml index 731af76..0273276 100644 --- a/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml @@ -33,9 +33,6 @@ spec: properties: location: type: string - #! providerConfig: - #! type: string - #! default: default version: type: string default: "13" @@ -55,6 +52,15 @@ spec: charset: type: string default: utf8 + firewallRule: + type: object + properties: + startIpAddress: + type: string + default: "0.0.0.0/0" + endIpAddress: + type: string + default: "0.0.0.0/0" required: - location required: diff --git a/packages/multicloud/crossplane/psql/ytt/schema.ytt.yml b/packages/multicloud/crossplane/psql/ytt/schema.ytt.yml index 1df865f..1e07cc2 100644 --- a/packages/multicloud/crossplane/psql/ytt/schema.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/schema.ytt.yml @@ -15,11 +15,11 @@ configurationName: "multicloud-psql" cloudServiceBindingType: "postgresql" providers: -#! azure: -#! name: azure -#! image: xpkg.upbound.io/upbound/provider-azure -#! version: ">=v0.18.1" -#! configRef: default + azure: + name: azure + image: xpkg.upbound.io/upbound/provider-azure + version: ">=v0.18.1" + configRef: default helm: name: helm diff --git a/scripts/crossplane-e2e-install-azure-provider.sh b/scripts/crossplane-e2e-install-azure-provider.sh new file mode 100755 index 0000000..147b22c --- /dev/null +++ b/scripts/crossplane-e2e-install-azure-provider.sh @@ -0,0 +1,24 @@ +CROSSPLANE_NAMESPACE=$1 +AZURE_CONFIG_SECRET_NAME=$2 + +up controlplane provider install xpkg.upbound.io/upbound/provider-azure:v0.18.1 || true + +kubectl wait --for=condition="Healthy" providers.pkg.crossplane.io upbound-provider-azure + + +echo ">> Create Azure Provider Config" +cat < Installing required providers" +./crossplane-e2e-install-azure-provider.sh ${AZURE_CREDS_SECRET_NAME} ${CROSSPLANE_NAMESPACE} ./crossplane-e2e-install-helm-provider.sh ./crossplane-e2e-install-k8s-provider.sh ./crossplane-e2e-install-tf-provider.sh diff --git a/scripts/crossplane-e2e-multicloud-psql/claim-azure-instance.sh b/scripts/crossplane-e2e-multicloud-psql/claim-azure-instance.sh new file mode 100755 index 0000000..873ff26 --- /dev/null +++ b/scripts/crossplane-e2e-multicloud-psql/claim-azure-instance.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +CLAIM_NAME=$1 +STORAGE_CLASS=$2 +CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} + +echo ">> Claiming a PSQl Instance" +cat <> Installing Test Application" +kubectl apply -f https://raw.githubusercontent.com/joostvdg/spring-boot-postgres/main/kubernetes/deployment.yaml +kubectl get deployment + +echo ">> Showing Secrets (1)" +kubectl get secret -n ${CROSSPLANE_NAMESPACE} +kubectl get secret + +echo ">> Waiting for Managed Resources To Get Ready" +kubectl wait --for=condition=ready postgresqlinstances.multi.ref.services.apps.tanzu.vmware.com/${CLAIM_NAME} --timeout=400s +# We can also wait on the "release" + +echo ">> Showing Secrets (2)" +kubectl get secret -n ${CROSSPLANE_NAMESPACE} +kubectl get secret + +echo ">> Showing Comp and Claim status" +kubectl get xpostgresqlinstances,postgresqlinstances From 7e210b70c2f263009b5f5c71ede157bcf9c93e60 Mon Sep 17 00:00:00 2001 From: Joost van der Griendt Date: Tue, 6 Dec 2022 11:50:42 +0100 Subject: [PATCH 08/41] small fixes --- .../psql/package/crossplane-psql.xpkg | Bin 0 -> 8192 bytes .../crossplane/psql/src/crossplane.yml | 510 ++++++++++++++++++ .../psql/ytt/azure-composition.ytt.yml | 18 +- scripts/crossplane-e2e-multicloud-psql.sh | 17 +- 4 files changed, 533 insertions(+), 12 deletions(-) create mode 100644 packages/multicloud/crossplane/psql/package/crossplane-psql.xpkg create mode 100644 packages/multicloud/crossplane/psql/src/crossplane.yml diff --git a/packages/multicloud/crossplane/psql/package/crossplane-psql.xpkg b/packages/multicloud/crossplane/psql/package/crossplane-psql.xpkg new file mode 100644 index 0000000000000000000000000000000000000000..55f1f33cb39a0c70583731ff6fa575acba18a825 GIT binary patch literal 8192 zcmeHMcT`i^)(^cG6#=PA7X;E1np6>`L{J3jDK~TiN$4O*5fQL~iqaGTks<;TK}3pl z1V%wc0i_Aj5fniXzN<6uTk~eU=d8EJS!@3Atdn)`IcML!&-vZGcd~z>x#8g`v?c}( zC8H5=6ch;%02CI9M36~v43ZWHb~?L=j;~G=>a^!I3aD97!T0H2@?D z3x#~ILBWPbA|YR|VEf}021OzMHTUZ}9GpQzc79*$_bmPoYyLDko(h`(A0GLSqTi18 zyTkoQp?}_Fs0^M;baSTzM7lo}kkOQp*&{=w0(d$=0teusV5;Bx5hzsiTekxR+}vq& z3N=_p(?0aigv0mzxrpN)84B%J(^LwDP6qEoW$3}aU$^6h4+f|-&?d>9Om=oBfot|M zv|pVTjsy^77=lP5kWnZ+1_MPwHDDSVSR#o;AfVB3fCM8W;V?2BiYLN}1T+DOfa5jD z00v0{hyWT3YW+6ow#Rh)6UVbaf)A7mmfkK@$iVK!SoJ4KkTX zK*EU#@DUUN0f3Ps;}{OYM)4uLyMEof8J+-meLICaMU6FSKL{vN@x$$@GHQR{`2Hiv`c>GU1^6G@L=8k0l_s;f!) z?IJkVrPZ*F@dd$6t*ma2O`V|fp#E@{Aei$W{x=W9>&J18IM(7)gW6QxRAFB!pE(?J z_U_{sUI`UgTyf(L_6GPi;o3dzsd+*bW3jJl?PeT>{4>jl)~I$N#-Wc+^2M#O+b5-n z75Sw0#**So2W})fdb+E1=rkWJP02m6JFy8{0?JPxr&@}uOqq!}RphExUQ+SNd8q7S zf=n7d+FQ$5FZe8XckD{RvM90W$y7mWuEmv=if3C}A`5e&a6sp?PERZE%1RE8%2{j2 z4qk?z0z$QTHGe}v*TS`TzRfT_MsyI>pT-!S1ZA+SQX8ZOb{LWH~kifUWn;yH` z$+E(&`P-&+aY6@YX6D~SEEx0=%}j6DWgK8>+&V7jWy9r-J^Zo2R#t34Wn(+S|A|`e z?c9r_OT4KrnMu-@YFi(NAmvm!RB~}9UB{XAzeo1S-U z)pRX)5+~bNqTF~U@*Im#d%Ca%?c~D7D^xkeGw$}7ow$(AH%y3>MJE)yhu|%V%JQcV=FJ#7 z3j24@kv^>CTkKZAMkY5!r=F6Z;n_VLU~zzjsYtkj33Ce19$D02>)5OI5Kg(eR!-Jvpv38F&e@vy*23)Lv10n~qzB)%jytYkFV>^u$M^eF5P_>5-N#b+fn22m zR{kpuU40w_`_^$SBFb-E5pjh#dOllSC~sh#R8P=UpYkucdF=A-n&hKGxi-&PYnGsw z*DPKeIbEbGQr$g^U$F+B~;b%YO4SLdzlvdS&*)2EioDOz;i42REdrR)w0lo;WQ zcdf*o3V3^VjAfX=a~K*PZ4)?z{AKY#)Lgzvz8u@u<${zKO=T(P>P?e483(kSZ>P*< zu+=7icoVwrYPIdS`tck#D3$Yz;{7`-q&)kroX~U$@2zuOu4O4#te?m=E1RgDvG%S| zHob*=t1Q_pN{@*xwX@BwwVBSgI7U=6m6-2ydF*gkPZ4;NnVxmba>8I@VxGTq^_f1l zLs|9tVo67ea$^1>+_H*;qdHi~(l;fKVeB!)%&{X+pjwS`KIn<#s6m6V4W9(PPa$~=zF6&2@kLm} zTvyaI*QD_dZjH$O5OG&?s` zImUH^(a{^8#2&BX$Qc;i8n<39i*YkQ-6V9FyIiBz>!SR1=4;XxFU_6}Dc)=n+dZ|7 zdpY9{^DmoEfUDjy=%>b$5aJbO)5o;x_xlCrGw)pPo`Z;YeUZr8%R4u?JZGE{v*k$Q z(b!8_M#rWY^!>c9=6>3HCUp(SUtafU%G*V_Y>T}ov37k`p}PW^*CWMI*42_FcrH>ZQmDuAyss=8{EmQ@y;gA)vs$|$qhjG&_?5=vH? z7b_k6J{#aJ)SKV7T|R{=i)q=3Zdkk8?VQ9)EBS;x@GK^fYTeua`i0c89s3#1nD~!p zd$~4zO1yln9*&K0clM|sO+*2O`?y@kobEmbHjD1Hh(7NeauVN_7WQSz&s+bQb<9BS zvQ}`kNk+xBa=F;Lh?OE`{9eO@4Mi}nr3N{(=?Cn=y*uMl_NfP@+KkofICos^E$hm$ zHL$KJgr#}Os`Q3Jp0$2XOBT?TOyBqDv8%Uo*y%@@E@KaSldh}-}hCf(ehGapB zQY>LtsWAjnD@hiE4P-Q&Nus~( zGK`gL?U0DE-D$M-HydFF zEszGIp-Fd-O?KV96#9ldfgZxNLQ}rZ(K85WHqYju{Gj~;q{0$vr9yc60+&Z!QnaKe zL#tOZVO^^oZLm$8t?%J6Na6|EE+*r|Q`#+N#wm*T^NV|13cOTA%+@|p)cMX*V`563 z6AOm*_X6jn9<3d!jPZFTcWrGb{<)w`ME0lpZECxa;ZYaH<~pbLihgOUVtx&M`_P80 zF?fsZme+#b8X#T~iRHeLpyX&)Iin6qW*eN?Tl}F4rvjm_PNdA}esZsRH7i|?<0;rn&CNGRbcg* z)%#GN$@_u)*x8a4P0wQohwY6XMH$)#oRpv*p=NqKowDR}R%28is~j4-HRDD(!QF7} zIHV(6Tlm9CJ=67PF;6kO`)&JE=@HU?dL?Pa`>gBccPw9?n|HfaYoFuw{;K-Q^pN;N zEWghk1J|^M_QwpnJl~dLnq|SYi0v+p1-VLw0!(_qm5wJO%|@P>N2`$A_;imFVjZ)n zzck<4;!y6{WmI6EB3f2@yMYYqD3?Ji0O3;mjgYU%oadN8{cxv+O{cp6(R8RWT?>sr| zY`WM8Z`rRaskJ!vncZvr62sMt|KLW0lh`G>L;6Z@pO_L@#P-mcayRRsj%gl>kAze{ z;facs%(9ajVh3eemaq3pt8zNU;zLT9GHkV|$2;!1eDG<;N$qc|JN6W>P3U+@s26cw zx-;|ceplUuByaQe_g;Q=n-nN-v1@|i1{uoR>6%co(jahn=UL=}u%*9oS>YEigT3|= zaSzlIRi`RK@5!9NeaNjclsqChs2kceTl9hF?yx|GV2zdziv!{{yNk=iL4+ zPmW|)j>a*+pOZi>WZ7oN?4*X8oO}SGA<}Wk@FQF%qZ3S@G6@fA)g{nQHOTec;c?~B zjR_y66I<_Zm2eo`6-J+cj4as4DnZ|l7}m1(v^?c$3`meQTH@XkLX%>PGD#ZLKOFXQ zG<_bwZw~n@%E0j9J-VYD^byZE{`wD4u-BY(C+{HOwWU+sm zfy@oxSnrfMJv+3~?`gWRMm@_~^j-%QFCS{c<9Rn}uIP|Gac8M*p>5N;cd6&+#3V1J zNsrm{(>p?!VB>9%`eigR_rn916cPyhq^-Lx^5604H5*BJY{?LBQ=A)EXn;V^G>-5= zBG>!Xq0KR}$#FLrv1g2xW4!~EZVAVWnLd=yyO!$>*E8Uo5SN&I&3uhf)*OE|4&euf zY!_Fs@>skPXIkfd;SJ7M=AEk`?!+< z8ePqUM)CPkO8=nePxv1w$WZ<}{{w@8T<*{K{{xJF%MfJn4S(aU|3!xX)&GJ(O4=v0.18.1' + - provider: xpkg.upbound.io/crossplane-contrib/provider-helm + version: '>=v0.12.0' +--- +apiVersion: apiextensions.crossplane.io/v1 +kind: CompositeResourceDefinition +metadata: + name: xpostgresqlinstances.multi.ref.services.apps.tanzu.vmware.com +spec: + group: multi.ref.services.apps.tanzu.vmware.com + names: + kind: XPostgreSQLInstance + plural: xpostgresqlinstances + claimNames: + kind: PostgreSQLInstance + plural: postgresqlinstances + connectionSecretKeys: + - type + - provider + - host + - port + - database + - username + - password + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + parameters: + type: object + properties: + location: + type: string + version: + type: string + default: "13" + enum: + - "11" + - "12" + - "13" + adminUsername: + type: string + default: postgres + storageClass: + type: string + default: default + database: + type: string + default: postgres + collation: + type: string + default: en_US.utf8 + charset: + type: string + default: utf8 + firewallRule: + type: object + properties: + startIpAddress: + type: string + default: 0.0.0.0/0 + endIpAddress: + type: string + default: 0.0.0.0/0 + required: + - location + required: + - parameters + status: + type: object + properties: + version: + type: string + address: + type: string + location: + type: string + binding: + type: object + properties: + name: + type: string + additionalPrinterColumns: + - name: address + type: string + jsonPath: .status.address + - name: location + type: string + jsonPath: .status.location + - name: version + type: string + jsonPath: .status.version + - name: connection-details + type: string + jsonPath: .status.binding.name +--- +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: helm-postgresql + labels: + crossplane.io/xrd: xpostgresqlinstances.multi.ref.services.apps.tanzu.vmware.com + provider: helm + database: postgresql +spec: + writeConnectionSecretsToNamespace: upbound-system + compositeTypeRef: + apiVersion: multi.ref.services.apps.tanzu.vmware.com/v1alpha1 + kind: XPostgreSQLInstance + resources: + - name: tf-providerconfig + base: + apiVersion: kubernetes.crossplane.io/v1alpha1 + kind: Object + spec: + forProvider: + manifest: + apiVersion: tf.crossplane.io/v1alpha1 + kind: ProviderConfig + spec: + credentials: [] + configuration: | + terraform { + required_providers { + random = { + source = "hashicorp/random" + version = "3.4.3" + } + } + + backend "kubernetes" { + secret_suffix = "providerconfig-default" + namespace = "upbound-system" + in_cluster_config = true + } + } + + provider "random" {} + patches: + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.forProvider.manifest.metadata.name + - name: password + base: + apiVersion: tf.crossplane.io/v1alpha1 + kind: Workspace + spec: + forProvider: + module: | + resource "random_password" "password" { + length = 64 + special = false + } + + output "password" { + value = random_password.password.result + sensitive = true + } + source: Inline + writeConnectionSecretToRef: + namespace: upbound-system + patches: + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.writeConnectionSecretToRef.name + transforms: + - type: string + string: + type: Format + fmt: '%s-postgresql-admin' + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.providerConfigRef.name + - type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-name] + toFieldPath: spec.writeConnectionSecretToRef.name + - type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-namespace] + toFieldPath: spec.writeConnectionSecretToRef.namespace + - type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-name] + toFieldPath: status.binding.name + connectionDetails: + - fromConnectionSecretKey: password + - name: release + base: + apiVersion: helm.crossplane.io/v1beta1 + kind: Release + spec: + forProvider: + namespace: upbound-system + chart: + name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 12.1.2 + values: + architecture: standalone + global: + postgresql: + auth: + secretKeys: + adminPasswordKey: password + connectionDetails: + - name: type + value: postgresql + - name: provider + value: helm + - name: database + fromFieldPath: spec.forProvider.values.global.postgresql.auth.database + type: FromFieldPath + - name: username + fromFieldPath: spec.forProvider.values.global.postgresql.auth.username + type: FromFieldPath + - name: host + fromFieldPath: metadata.labels.address + type: FromFieldPath + - name: port + value: "5432" + patches: + - type: CombineFromComposite + toFieldPath: spec.forProvider.values.primary.initdb.args + policy: + fromFieldPath: Required + combine: + strategy: string + string: + fmt: -E %s --locale=%s + variables: + - fromFieldPath: spec.parameters.charset + - fromFieldPath: spec.parameters.collation + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.version + toFieldPath: spec.forProvider.values.image.tag + transforms: + - type: map + map: + "11": 11.18.0 + "12": 12.13.0 + "13": 13.9.0 + "14": 14.6.0 + "15": 15.1.0 + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.forProvider.values.global.postgresql.auth.existingSecret + transforms: + - type: string + string: + type: Format + fmt: '%s-postgresql-admin' + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.adminUsername + toFieldPath: spec.forProvider.values.global.postgresql.auth.username + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.storageClass + toFieldPath: spec.forProvider.values.primary.persistence.storageClass + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.database + toFieldPath: spec.forProvider.values.global.postgresql.auth.database + - type: FromCompositeFieldPath + fromFieldPath: status.address + toFieldPath: metadata.labels.address + - type: ToCompositeFieldPath + toFieldPath: status.location + fromFieldPath: spec.forProvider.namespace + - type: CombineToComposite + toFieldPath: status.address + policy: + fromFieldPath: Required + combine: + strategy: string + string: + fmt: '%s.%s.svc.cluster.local' + variables: + - fromFieldPath: metadata.name + - fromFieldPath: spec.forProvider.namespace + - type: ToCompositeFieldPath + fromFieldPath: spec.forProvider.values.image.tag + toFieldPath: status.version diff --git a/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml index d62c228..813b988 100644 --- a/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml @@ -106,7 +106,7 @@ spec: kind: FlexibleServer spec: providerConfigRef: - name: #@ data.values.provider.configRef + name: #@ data.values.providers.azure.configRef forProvider: resourceGroupNameSelector: matchControllerRef: true @@ -119,13 +119,15 @@ spec: namespace: #@ data.values.crossplane.namespace location: "" version: "" #! 11,12 and 13 are supported + writeConnectionSecretToRef: + namespace: #@ data.values.crossplane.namespace patches: - type: FromCompositeFieldPath fromFieldPath: spec.parameters.location toFieldPath: spec.forProvider.location - type: FromCompositeFieldPath fromFieldPath: spec.parameters.version - toFieldPath: spec.forProvider.values.image.tag + toFieldPath: spec.forProvider.version transforms: - type: map map: @@ -139,7 +141,7 @@ spec: toFieldPath: spec.writeConnectionSecretToRef.name transforms: - string: - fmt: '%s-postgresql' + fmt: '%s-postgresql-admin' type: Format type: string - type: FromCompositeFieldPath @@ -147,7 +149,7 @@ spec: toFieldPath: spec.forProvider.administratorPasswordSecretRef.name transforms: - string: - fmt: '%s-postgresql' + fmt: '%s-postgresql-admin' type: Format type: string connectionDetails: @@ -172,19 +174,19 @@ spec: kind: FlexibleServerConfiguration spec: providerConfigRef: - name: #@ data.values.provider.configRef + name: #@ data.values.providers.azure.configRef forProvider: serverIdSelector: matchControllerRef: true name: backslash_quote - value: on + value: "on" - name: flexibleserverdatabase base: apiVersion: dbforpostgresql.azure.upbound.io/v1beta1 kind: FlexibleServerDatabase spec: providerConfigRef: - name: #@ data.values.provider.configRef + name: #@ data.values.providers.azure.configRef forProvider: serverIdSelector: matchControllerRef: true @@ -203,7 +205,7 @@ spec: kind: FlexibleServerFirewallRule spec: providerConfigRef: - name: #@ data.values.provider.configRef + name: #@ data.values.providers.azure.configRef forProvider: serverIdSelector: matchControllerRef: true diff --git a/scripts/crossplane-e2e-multicloud-psql.sh b/scripts/crossplane-e2e-multicloud-psql.sh index 277903b..f44c12d 100755 --- a/scripts/crossplane-e2e-multicloud-psql.sh +++ b/scripts/crossplane-e2e-multicloud-psql.sh @@ -3,11 +3,11 @@ echo ">> Local Test" export CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} -#AZURE_CREDS_SECRET_NAME=${AZURE_CREDS_SECRET_NAME:-"azure-secret"} +AZURE_CREDS_SECRET_NAME=${AZURE_CREDS_SECRET_NAME:-"azure-secret"} UXP_VERSION=${UXP_VERSION:-"v1.10.1-up.1"} CONFIG_NAME=${CONFIG_NAME:-"trp-multicloud-psql"} CONFIG_IMAGE=${CONFIG_IMAGE:-"ghcr.io/vmware-tanzu-labs/trp-azure-psql"} -CONFIG_VERSION=${CONFIG_VERSION:-"0.0.0-0.0.11-10-geabb607"} +CONFIG_VERSION=${CONFIG_VERSION:-"0.0.0-0.0.11-13-g80c32a2"} CLAIM_NAME=${CLAIM_NAME:-"postgresql-0001"} TEST_APP_NAME=${TEST_APP_NAME:-"spring-boot-postgres"} STORAGE_CLASS=${STORAGE_CLASS:-"default"} @@ -25,7 +25,7 @@ pushd $(dirname $0) echo "> Installing required providers" -./crossplane-e2e-install-azure-provider.sh ${AZURE_CREDS_SECRET_NAME} ${CROSSPLANE_NAMESPACE} +./crossplane-e2e-install-azure-provider.sh ${CROSSPLANE_NAMESPACE} ${AZURE_CREDS_SECRET_NAME} ./crossplane-e2e-install-helm-provider.sh ./crossplane-e2e-install-k8s-provider.sh ./crossplane-e2e-install-tf-provider.sh @@ -33,7 +33,16 @@ echo "> Installing required providers" ./crossplane-e2e-multicloud-psql/install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} -./crossplane-e2e-multicloud-psql/claim-helm-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} +# ./crossplane-e2e-multicloud-psql/claim-helm-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} + +# ./crossplane-e2e-multicloud-psql/test.sh ${TEST_APP_NAME} + +# kubectl delete postgresqlinstances ${CLAIM_NAME} || true +# kubectl delete deploy ${TEST_APP_NAME} || true + +# sleep 5 + +./crossplane-e2e-multicloud-psql/claim-azure-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} ./crossplane-e2e-multicloud-psql/test.sh ${TEST_APP_NAME} From 4590ce2411debdd2d34cc4bf708a030247777536 Mon Sep 17 00:00:00 2001 From: Joost van der Griendt Date: Tue, 6 Dec 2022 15:26:25 +0100 Subject: [PATCH 09/41] testing --- .../psql/package/crossplane-psql.xpkg | Bin 8192 -> 8192 bytes .../crossplane/psql/src/crossplane.yml | 8 +++----- .../psql/ytt/azure-composition.ytt.yml | 2 -- scripts/crossplane-e2e-multicloud-psql.sh | 2 +- scripts/crossplane-install-uxp.sh | 2 ++ 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/multicloud/crossplane/psql/package/crossplane-psql.xpkg b/packages/multicloud/crossplane/psql/package/crossplane-psql.xpkg index 55f1f33cb39a0c70583731ff6fa575acba18a825..ffe3e7f0d0d87e7139a288ebaeada4c2547262bd 100644 GIT binary patch delta 3577 zcmbuCSvb@U8;57gGDFfL*=5SU&At##B0`F+sm7SaHp&(a@ek8XmL!s~4G}L{UW|PS zFJ)^egb*c(HX=*ezRupW?>qROv*+ag{jTS_o`d@q@fOiM(36bBU=7^eDeeRWnU2KM zXfzywLdW3HbUK!RrP1hQIGlz>Q!rR09)ZS^X=FGBK}8ebI3$8V!=Y(-GM#?^k>Y(S zQWSwkBSA;t$z(8<4+b}aiC6*+g+|caF?1>cjig{HG#Zjl!Juhy9L60*b;n>R?oNz$cA$b>G;eyS9o-_2omx z8h?L9hYSMF6>}-%N`t<50yZbMRd5WQtW_^1&%hP`L6*IA=_-2~f z_H>#XHbXF}pqv4C@LfJl<>B}v9K6V)J2qPx2w9JM!HdZTdFQ9yM1-Mr-%U&b6WKz} zDdE$0wA^@TF)|{L2?aFMEgspY@RJ6u6&SP0#U<910IgWdLTiuQOsPx!2VTCSFxmozFF>IFlU-;v+_#w$`}7p3Q&nMLmTh;cy?=eT+Z~6L z-1HQZcY~%es@n?|3sqs++X0Nq2ff^K{puO|p1O})=rN!uL!N4B=z#U z;yjD8*XBPY9sL|CSe=i{q;vHKE#7JgBCV%u1ScjPH#7CYlHeH3K%;iQXa=FT7?UA? zy7c@`xLSON-uACSQFRtnB}v=1m!*9~dcQWgIo_{5MkDo2PYxms17w>NlzkRXRUK&p zcuEI5Dt;_a<9MDNk$f6ib4Ir3%~_C!bDD1bz_xQ&`aMVdy7tM(1f80sdt0Y#p?QJ0K5|`+85%R-7(EENUt(=` z(&qG2wGJg1L1hpFT==51ztW&Qu3&c%7&{CjckeeUFmGSpmT}(dq`LXFC23zNppJRB z2eDjfjQ4i0duhSDOF)tuVC`2LAFUieg>p5|+fh!@5v|ZP_At5$uC1TkxJrt9^c~#xPTHVLTj%8{(<#=3eAzuS{S@B~{wto;+Fk&zFjr?T!)#e=pz0*HPPwxW&+Bs zp!)2+;^4$YlgHQl8r7mEMOAOaFl@T3wni~onW@^%BGt^muQ!OGE69h*2P>)dCVO>* zemV{{JXyg42J#O_96N0?g|VBqGYOVupB1)bqoP26@gRQ(Wc-JaJ@Kq;w+&ss5E z=D1CvMY-73(>l79ODV4)raLpIIoNCMorJZT7&LGqHY}rFK#DjU6{OMa;%MsEB@_JH z_?_M~VEilzmC)Z|5W{Kl04^7%lY8_R)D;F^#XZzH60ceJatP29C%i2RAs2y)h6ERiAGO zS=X;K-L%!<_cdqUo;an9&ZND8cz+cN!*Nhj)>`V2> zJ(n@(YU-v*FR54b^lKI<>r8n84kZ!T+s8N`{&ZziiQb=U;L^># zDI(e4b`&Kuai(DT!=tv5?IQ*`8y5LzxV8CXAI1*eC<9mgJeaPS7I(X?q;v1Z_|xqE z8q&^O?fCl6^_@Jh{8v172^j#xrbvomj(F@jI@d06L%lJDpjgZ0(-e^mri}dy4V%(K zsjatVkEunE1_RouG2y}Y5 zHAz`n*3k3`Ij9;(=g-JeN(9B1&7SQGdwR=<&z78z{-C4a$FNk^^6(C4UKYP=A`Cy= z#kDqre)n}QiL^1wbDKg4KwGtXgH*xeX#>b4>lm#6tm9bx2_`(4m36a*a|!^ypUU*Z zN#VvD<*!Szou*vqNdTRx_hmu}k%?iGT0lEO%*8PCFA84a?P_BB`oB-Dgr{4U+25>kFxNAA@GkeuBg6Jypyk7m;;bX>{;thME zx+K3%JgVm6J#QW%>C%vw&if7A^i(0fzS(iYK+bqkvmMB*V+%z?y#Zf0jVp` zXI_)eVNnKv)u;2wF?OkLuPJn*vC=H=h9K}pgNjiTCusGFIybC2!oO*w+R=*ITrvT^e|s0vWfBV79u^yZzb zL!6J0!-bX#r6&1KEqqtLzX$cPhc++VJF(+Hl_5IzI?KYR`5$*Ypxo(?YaYtRikF4jJ7~WD0C<#s(xFs?4{S-H#e`7pHGOcdnf*R6~MnXnjC9r`rcRcn;S8=%sVh0h4G6Ag{Tkex>bWB#{1DG zb%wWf<-IDiI>q8rQ(G>*yr{n=CLPr~r({JDYE<^Ay)Og5zyiM22=AMPwo^gDJh5Ab zOWYTQzZTrEITTO?Y;hBvABI@C5G%KH0B+C(+9XIfHP5di6{l+k`_22uF!pbk|raDKc@$CYcT1yS(ZB^l= z!2^C8VHS{Iz}c7synf2Ac|_Cfy47OA-CYfmJU0ngfO@F%@$CbqgK8SK-X!k#=vP^p zrFGXYd=H~NY!7!x>Pv(l8~d#_cv;^SmCZIXL3Z60M%(bn-gaI!Bt-e(ALErHy-)- z0%KLUnBkxq!7fl8TrIy=j1)ZK8d&iHXXOmGqM}K`Y=ugv0447tZ`tu7n4OE5QbKRoIgY(I z>b((DAi|!i^VOE|I%9n38BCjy!}v|V%MD&plG}`RL~3)J@)u2$^fZ#Sn;e{A{Ym4F z=7#b{A=Sssbt-OF4W8XmAia$eeD$=$8X3Ks>asG+g=j$tCzvq|V)X47IX@R<3w*VF z0jf+!PxBNTT1NE0nZO>f;Dl?eh>ABkk-3xJ#G}LP%<-kY@qL22?k$aZg4OZ(l3U8Z zz33aJT@RlxWM#Q86S7+JBi_8i#zpoUevUuYGP-$ZJ0G=2k+MwrU4YtSL|GmQ`p!lq ziptfUvcG$u@kLHPvH#A8ceu0xm-@;Dz)@o{G8h?Fm?f6G#dnx^ZrNfoc+*MzZb;`R ziHWY^>@qw`>Av;5VWj}O%OYDL_cs#d6H(yru;v|kp3W{2)o(S9|w zH|NQGU(n{5AAgc)4D&S2=ktjSDTz@N<(_P>Qi>Ax!wzg+IT@P7crJ!8iJ delta 3602 zcmbuCX*d*&*T!dTW2Xq&voB%HZr}H1qGVrYwUd1}vZO*KDzcS~BodQ}5GMOtp~qTd z$(9h3-M{~r_x<+1AD;U<=fk8Nw3{Suj5U89Zh6D@PUkey40f!?%;TSL$hSbo&Lh&d#9D#r#QE;LL zLgT+e{WStdVbq~0FcgeLAn`~ngzz^MPD)b>iwPr0UCi-dBr+x>Bv8(Xi~;rk?b#fC z784-^5&-^R0l*M25~lt?F95LvNm_xy(}>UeYn}!dc^7h)XU1&~ST`B@hsdmxO$Tdl zqVz++4QHbq>t*_6zK{R>-6}!s$u_rWqT;mv5!VK}iQO(d&%KoOVZszIh*jCos^}9$ zGi~q3pDb?jo@$j0gHCm#DbZ62jM3_z7rDQ^PiS96cc7W7YYkiT^zsC*k;H@6(F~=E ziT5Y@D{b?A*kb|-T&EE2k-ofg{`!Tq$yS#&cm9atI-CQ%kDq$!w})(XPujT^aa?@` zzOSRE`k}?MOm}}@)qb5W%i3J>b;Zn1RL$T0ABmxMBFd}Q!XEWxm4=7PL1piiyv$)) zvsXu3sYjIuGH(|0D|dxRxT;U9l|5wJ{Jr|Iv)xHA$zTXU=RjwqhjVYQj6?aZgL^+G zHAD`oQoUbsBB$gloZXvSKWJYcQJg1&lfBn_ME%}fU|QUFj*gM6wT+GX9&_q^Qq96X3`eh`Ri{(n*>mJgt0k4jS=EmHdkQ zV1Ao3&#O2~;$dsgKrBo~1*A+yn-3*2=*zBRWwW@tvD|BZWfo@$AX`r3*@kaI1_N0s z*8Of@sJvv0iQ!ZBafWN3ee08;wiB(OlJgXM|62QZ*w(D~*EiLs4Ev7?%L3(|tCyP; zy}T_JxId7mWgJQBAc|lRCgtIj4S_+qWd=8mj8s|B(LD+4;ll*%?~ORV6sv6a9!+nG zhX~QN0q(=GRPJ7V+uw^hT9OTQORmxF7WK`D_4WIuT(j6DX+%b~Z!E^H#GBw>HM(?% zTMK*jtry5hC^%l*P@9r32tORO@k<><43!C4<+b?Y`$%8?9kOvb8ywDy^0c641xFW$ z8K=IUm0#4JW6qb!8&6(ng9U97w5A7=Wjar|No7@My5j98F;m3ew-ecvCnSZ0>vMk! zfzXyf7PsFinGQ<9RvNG~Te>psVc4)P;&F-5h$5kdDsrg+nO+QHpp$uJwXn)|!gC+n z^@AP!Nd;flOT%RQzO0OAziqptXFDCDV&i&WPa4ci4+vov(69$Tm*y^k7l;^M z$M5V_*ec4Ql5#pz@@~qmaVXA4+FCFIs|4zS$eS3Jq$-+|`>5)BKm+N-IiNDFG~~CgbkJ@W|n&v-S&KaoT35-FIfbU@~ED>fSrS9KEuT0JhOwoDx3T&Oof2oqc?2K32AHo3X-}kDU}l zbsDfgBB6!7sL@#|HS_I^qM54ll+M{vy5p>5OzGjx=lz9I->0X>6hbk4D?=zszB$Hi zYt9SuDge2zsjaGv5CVi?pqsy~2gXt-T6M#S|19jT^Q1TD9 z71Q(V<|$3jc^YZ<-2^NwrAA- z(S$6PTRQHnQ87K~M-*wKkIn5){>$tXjn;q%vQHVFNZ3wTkJ06ycM2=6@{!ngi(WH) zJpDv?9GHsuY`Ox#9_Ja&AXa-faOHi#n3vs*sF#bEDJU)uWk=GuoH$<)ly0nGIZv%sSI^s;&eU zx0zJLmG=^)e@q`;`Y>A7&ukyb)sfuPcp6qxmsW=6jTbMfb37%5!-XJJwv(Wm zfNS>e7v|UpN7Sxn!U-=;*t{1!-V6{@RL#MorEd580mYnkJO{G+n-z?YL}sLZ@eY$JF;kj^;(X+g0d?C9G`ykR$hW=vo4 z^*d$JE6)u-k;R6`0{6m{o`ObZ2%23#%itm4DIWaGn$QNhgu+cWzqYItF@LJos9464 zRv*HUPlS2=J;@o6d0l!4XqtIbyW83{SN>f^^=NlxfU=PP9s_w5W2sVyGGLcXKz$;9lUk$E}K`26k>u`%s zzMGh)?(-vz#?<=#Pez>Btw;Pf>TX6uDp>D)gpu9_(VxV<`u2P8E{rSc^IS_b$~=YV z9n^W)dxaWg+7TT6EW>hHgc~ihf1jhJ+%@7^pH$OIMEL7Zmp!evIw0L< zdSWE4t%U=YfW?PY?*MMg6PkzcLzznk3g16jVHt%lh6Bl`ZD98TKlu;*%6~98`345* z6%AoaX~x~BqY^5t9%-1^8eox=R%l}XTd$pHB)7^ zKJKm5b@UFo(MaqH?~Gnt=g+DgjyJPB^}H=wI>8@JBTt073VsRn8VCIhv-Ruo+EWdc zIO!RLZCYK#5sKPzy1SEeA{=wEoGj@E3qV@Nr38WCS5LL7s=1p(y;;}I&V0I3+AvQ_ zXZW=)3g0YoDovFPjWfT|3Na6a~ zcV|W2W-i2sF9GH@UDFi6-{*{4nMS%lb96*zNE>gnpNS0KQUjpfiXEm zZ}F{oN8k8crxtr?003}$itn!(V*nUae)Ea5kp3-VNnMn(CtM4@g7FTM(hQ0S2)HO^ ajfo+I{+(Q<{(qJc|GycD@>=*3^M3#(lY{L5 diff --git a/packages/multicloud/crossplane/psql/src/crossplane.yml b/packages/multicloud/crossplane/psql/src/crossplane.yml index 09fa22f..cc52949 100644 --- a/packages/multicloud/crossplane/psql/src/crossplane.yml +++ b/packages/multicloud/crossplane/psql/src/crossplane.yml @@ -124,7 +124,7 @@ spec: toFieldPath: spec.forProvider.location - type: FromCompositeFieldPath fromFieldPath: spec.parameters.version - toFieldPath: spec.forProvider.values.image.tag + toFieldPath: spec.forProvider.version transforms: - type: map map: @@ -138,7 +138,7 @@ spec: toFieldPath: spec.writeConnectionSecretToRef.name transforms: - string: - fmt: '%s-postgresql' + fmt: '%s-postgresql-admin' type: Format type: string - type: FromCompositeFieldPath @@ -146,7 +146,7 @@ spec: toFieldPath: spec.forProvider.administratorPasswordSecretRef.name transforms: - string: - fmt: '%s-postgresql' + fmt: '%s-postgresql-admin' type: Format type: string connectionDetails: @@ -158,8 +158,6 @@ spec: value: postgres - name: username fromFieldPath: spec.forProvider.administratorLogin - - name: password - fromConnectionSecretKey: attribute.administrator_password - name: host fromFieldPath: status.atProvider.fqdn - name: port diff --git a/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml index 813b988..e916a31 100644 --- a/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml @@ -161,8 +161,6 @@ spec: value: postgres - name: username fromFieldPath: spec.forProvider.administratorLogin - - name: password - fromConnectionSecretKey: "attribute.administrator_password" - name: host fromFieldPath: status.atProvider.fqdn - name: port diff --git a/scripts/crossplane-e2e-multicloud-psql.sh b/scripts/crossplane-e2e-multicloud-psql.sh index f44c12d..995c2b2 100755 --- a/scripts/crossplane-e2e-multicloud-psql.sh +++ b/scripts/crossplane-e2e-multicloud-psql.sh @@ -7,7 +7,7 @@ AZURE_CREDS_SECRET_NAME=${AZURE_CREDS_SECRET_NAME:-"azure-secret"} UXP_VERSION=${UXP_VERSION:-"v1.10.1-up.1"} CONFIG_NAME=${CONFIG_NAME:-"trp-multicloud-psql"} CONFIG_IMAGE=${CONFIG_IMAGE:-"ghcr.io/vmware-tanzu-labs/trp-azure-psql"} -CONFIG_VERSION=${CONFIG_VERSION:-"0.0.0-0.0.11-13-g80c32a2"} +CONFIG_VERSION=${CONFIG_VERSION:-"0.0.0-0.0.11-14-g7e210b7"} CLAIM_NAME=${CLAIM_NAME:-"postgresql-0001"} TEST_APP_NAME=${TEST_APP_NAME:-"spring-boot-postgres"} STORAGE_CLASS=${STORAGE_CLASS:-"default"} diff --git a/scripts/crossplane-install-uxp.sh b/scripts/crossplane-install-uxp.sh index fb018e8..8f09a85 100755 --- a/scripts/crossplane-install-uxp.sh +++ b/scripts/crossplane-install-uxp.sh @@ -6,3 +6,5 @@ CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} echo ">> Installing UXP - Universal Crossplane" up uxp install --set 'args={--enable-external-secret-stores}' ${UXP_VERSION} kubectl wait --for=condition=ready pod -l app.kubernetes.io/component=cloud-infrastructure-controller --namespace ${CROSSPLANE_NAMESPACE} + +kubectl create clusterrolebinding crossplane-admin-binding --clusterrole cluster-admin --serviceaccount="upbound-system:crossplane" || true \ No newline at end of file From dbbb06dfb6ec87a060c1e3c572fe18d9af69b6f8 Mon Sep 17 00:00:00 2001 From: Joost van der Griendt Date: Tue, 6 Dec 2022 22:18:21 +0100 Subject: [PATCH 10/41] minor tweaks to get it working --- .../psql/package/crossplane-psql.xpkg | Bin 8192 -> 0 bytes .../crossplane/psql/src/crossplane.yml | 508 ------------------ .../psql/ytt/azure-composition.ytt.yml | 10 - .../crossplane/psql/ytt/definition.ytt.yml | 2 - 4 files changed, 520 deletions(-) delete mode 100644 packages/multicloud/crossplane/psql/package/crossplane-psql.xpkg delete mode 100644 packages/multicloud/crossplane/psql/src/crossplane.yml diff --git a/packages/multicloud/crossplane/psql/package/crossplane-psql.xpkg b/packages/multicloud/crossplane/psql/package/crossplane-psql.xpkg deleted file mode 100644 index ffe3e7f0d0d87e7139a288ebaeada4c2547262bd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8192 zcmeHMc{o*T+uzD8X_P6lJ7%`M_Pmuyl!J(jN!uRQHf`fJvxqXm}lV- zp-rVCL`jAQL{XXEUFW>tb-wSr-gDmT>b&Qlx93{dwbpuu`}aJ*=eh57-M>Y5BqGol z4HAh=!UIGq0tbR17Eh+4u_!7PgU5g%l?aD}7!(lCMhqZ8fR z-HQR$unqXL&Vj0b-o;K8>Q4U^+RNSDo630(fO<1LzD$QkWPo0D4kpEkN+mc^IBm92 z`mcu;g9ni)03@NQ6g&z+#*je}K_#P6ARLP(At@v@noOdQQB)F&h(ck>WDtX-V6Z4G zh$bSbND#%b`kN&x=e&a`5I|svWGV^{AW2v<3WGsmuw)_@i@@O!Bq|ySfD}0A4Ns+# zNgO;RX9x!egi|q4yFd92Jg4gIEljQwQMiBq|a^1~3RR$6*0D zz~RQngXHexM&T^uU&D~$7zBxgg@aT8Ma3Wh3ddp5cnls+0jUTi9F9igC>RO`j=*qy zo=hQ9hyaHz01roVW=ElZ*#Z)g4nhM11OKKI`5or(WZ+xR2aFRvVGjO(IOQ*)@ekrZ z8jJjY@gITx9{>N4)4!@wCX)ZFDejFok}mF&SVZOr6@IS6ht$-b7^m2_@a^i&IZk+G3XZtH}= zaWgPGN}v!Cl*14RR!Y&oZk|vP*KZ_6n~X0kG9r5^M;PQAIb^5pI$d$qNz{a2$)=V3 z%dLlT#}ZEX%hlEQDljK#t#qD~`~se%cTn=!w-ZI?DG35rmHG5^)2>8w&$=%AOBM;) zDG3DUdZk0iE+=S&fGpG8j8=ZNC#h7uYJ$3@_7wa!`9XGOeuP3xr$e$+kaBmNLTR*6 zhJMMTWA8*PJuJ$Yoi|EEvtj-EPnFphjK-_kTBDY$Y3f`eT;b>k4Jv(nsra5kbgJah z;u9Nz@=@)u^-ukL3QPgnI2F?#rpi9i?JC5kD37*q#l$DwSwH|fQzx(F6;GnvKx1%* z^v?FOcZ=iL3`Z7kE=W|G5bu6+9HMBQtXB76-TKajGfOC?ELv*>mr`1_GzK5xo%;iqsv(Z50QH!FgmFT`)&xFV2J<)NoJWBD+fDN8Hsysggz(6S{+@DLE7h4 z9lu=Y8yl;0!>6}FK4g?n?rb>CxT|t)5S^ZusM5q+#h|}As|`7WxE8;&oKmN=Rn7ao z<-_`$A21q{*9I&*jnjBAtELk%1|_eg*2F_XAU{Ea7hS2*j za&VnV_FTq~rP{SsWeGnDbmH}dx$ng#2##b4AR&&s4SI?y-|@7p&8rDcUC4H-7m zNsaym*_frHs%qs63HSMRHztmTEqSICWDX5vD10^=s4uH{7ezTKaS_{K1g``;~ z%vDa$F8OIR!#2LtY)=hsz2Vu=OIi{vyaH5EhKXwT$yO4cU%V}cWbZIJzdm6f?_pB=3~y4&FTdSP+sjkgKus@$&gf0_#CC`s zOHxux7F|#%>+Vi6f2DzaAuP$vFO-;WpI}w=2(i1bcD-MCT2?DoC=T#H*`J0>?`pe^1(j=Umm zt^1K;BR}RXzPR2xu)a?tYehfz$E2Fvp%+6t&z5kNz2BL#H#zcRYf22%H+(j7xFp};FbfDTC?-Kf5WUY~%l_#$wmtHREdvbXcKDH$t+(mjZyx~e6gc`+y|Q?jy1e(Sxwe8x zMXOLq^{LCw8Th!<51qINu1%WCz1*#S@&jGqWR5(obAtC01M9T>4zfAmVTV`ZhmMIy z1QQHWBiL~21Y#(xSgl7_V5Fg3FY+v#ezsm#vytV!bW3s7jFN``i+v|W3>XZdEGoU_aIN6*;+)p za9f;gF0;BP^Qri-bMbAQp*jB|tLG;4ZY)%*HF9+qY~l5kaj2AY>=C1oWlX%)jnU%M z3v!8#MmSq-;l={~LJC_bXBm<`{&Lwt=Fna{?zVQ&^_O5AQax^WqQ;9b5$=P1qO|S- zn=L20WVs=(0KfTurl~jNg(6qVa6KI^Yu!HjQl4AkJzue$^@R(s9XtwX7ZQDdsDrOY zZEVl~Y}7Hr_t80)<0QeJHKD|a1G>*$`QF-VXO}qBQ;=wnD2SgzznXm&Bxtw~rBi$0 zqMD>rd3vW{WMX3T>3gTt*91jFdZwif$=nSxE;U!g;3t`Ht9iCf3baumz8Qk&4=g0z z&HprKkI5oKf}r!OsN4Wx?qr0-#tyhfBV-jX+SoL+gxEV?{Xhbj^lr2G2Mf`GtjlEB z;SaTCr69ZZ$kXyWP5x2aB?c8+K2o`iw!HD&dXLP^E$E_H2gz5M(HY7M^`)(qfyKTL zJQM@;`9B>G-;vQr-ZTR=PJT3;&x_tvFOp#|4E0j*mpQn;L$_aE(bSocG#h$9J*~Lb z=j3bvc&#naBDUMVuYgcfEO<$hh}xbTE->(x$G&uH!`#h?WjhP$te)m}O7NHk=u$IA zwMr=Smzf_dRdO7&M44>&4+N-gMF=Lel?f6R%3f- zt}mxEJl~AiCmTIZo(wOMyZk)HHcxfWnHr@Q2bbK6 znwDC3wUk=r59{%%2YyTqH{O}CTmGY34WVKn zlA3;1KO9pnJ6G9%d_#)xG=%-e(F~)hd4KGVpGCy965l+(%y?8h4{hWB7tFkLa zEVa97EKHz8+w)vZ*aLl5pyCHUp++lW(x|ic{>Ncy!wXxZdfByIYl<^?!-G*p=Vd-S zQCD>DT>dNOV73VMbA_DT$1+&*GEJivEnsxfmSqVk^bxsM7 z+`q|NWa=_)Xn@m;c3?MD7B%-JVJ_F}>SVF>~DxbxiY8 zDIeHcX#SH+KuCgC-pygH%ro^qrn?$z6YVDb3g&el1?(mz`s)$k7V>2T;NA1*p1pOi ziGzjo*}hjL!A_EV1DU+bZy>YJb5_~Vh_}zyhg}TUSG+EB7roO&hsy=%3%OKA&la7q zCGRVRRUVIVzf_IOF{G%4_j}MPRb4Z5XP7tq=5Dl#e8BrR-2L&Yu?6?$DKX z-kvVmr7$}({|f{j(>^W?Ilnrr4)0*@k7wmnUyM1be9_fcu|PCj!tj<{URJIvLR&|8 zN>XZObW>J!c}I8(3+ah~?3R=_b6&_{8Me5_jhWD@DX-UFNUijHTFIMBnN}z0Z|;<8 zD`EYzT9LlM8c=_m?bo*~dH;b&o>df7deR@!amvMLz-RKxxp<*Hu!Dzh@qKZ?tv6~? zA-|^8-?TS>$6!~Yn-dkJd&8XR?r#4fz(3J{;2ah7f9wAMfaO$u*Z==R`|*3T90mW# zZ@TqA#qe+a7sSz{%|H)#bD{%B0(bMF(NsBt_*WI5Bi;W+dLswpFZyzEq=v0.18.1' - - provider: xpkg.upbound.io/crossplane-contrib/provider-helm - version: '>=v0.12.0' ---- -apiVersion: apiextensions.crossplane.io/v1 -kind: CompositeResourceDefinition -metadata: - name: xpostgresqlinstances.multi.ref.services.apps.tanzu.vmware.com -spec: - group: multi.ref.services.apps.tanzu.vmware.com - names: - kind: XPostgreSQLInstance - plural: xpostgresqlinstances - claimNames: - kind: PostgreSQLInstance - plural: postgresqlinstances - connectionSecretKeys: - - type - - provider - - host - - port - - database - - username - - password - versions: - - name: v1alpha1 - served: true - referenceable: true - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - parameters: - type: object - properties: - location: - type: string - version: - type: string - default: "13" - enum: - - "11" - - "12" - - "13" - adminUsername: - type: string - default: postgres - storageClass: - type: string - default: default - database: - type: string - default: postgres - collation: - type: string - default: en_US.utf8 - charset: - type: string - default: utf8 - firewallRule: - type: object - properties: - startIpAddress: - type: string - default: 0.0.0.0/0 - endIpAddress: - type: string - default: 0.0.0.0/0 - required: - - location - required: - - parameters - status: - type: object - properties: - version: - type: string - address: - type: string - location: - type: string - binding: - type: object - properties: - name: - type: string - additionalPrinterColumns: - - name: address - type: string - jsonPath: .status.address - - name: location - type: string - jsonPath: .status.location - - name: version - type: string - jsonPath: .status.version - - name: connection-details - type: string - jsonPath: .status.binding.name ---- -apiVersion: apiextensions.crossplane.io/v1 -kind: Composition -metadata: - name: helm-postgresql - labels: - crossplane.io/xrd: xpostgresqlinstances.multi.ref.services.apps.tanzu.vmware.com - provider: helm - database: postgresql -spec: - writeConnectionSecretsToNamespace: upbound-system - compositeTypeRef: - apiVersion: multi.ref.services.apps.tanzu.vmware.com/v1alpha1 - kind: XPostgreSQLInstance - resources: - - name: tf-providerconfig - base: - apiVersion: kubernetes.crossplane.io/v1alpha1 - kind: Object - spec: - forProvider: - manifest: - apiVersion: tf.crossplane.io/v1alpha1 - kind: ProviderConfig - spec: - credentials: [] - configuration: | - terraform { - required_providers { - random = { - source = "hashicorp/random" - version = "3.4.3" - } - } - - backend "kubernetes" { - secret_suffix = "providerconfig-default" - namespace = "upbound-system" - in_cluster_config = true - } - } - - provider "random" {} - patches: - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.forProvider.manifest.metadata.name - - name: password - base: - apiVersion: tf.crossplane.io/v1alpha1 - kind: Workspace - spec: - forProvider: - module: | - resource "random_password" "password" { - length = 64 - special = false - } - - output "password" { - value = random_password.password.result - sensitive = true - } - source: Inline - writeConnectionSecretToRef: - namespace: upbound-system - patches: - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.writeConnectionSecretToRef.name - transforms: - - type: string - string: - type: Format - fmt: '%s-postgresql-admin' - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.providerConfigRef.name - - type: ToCompositeFieldPath - fromFieldPath: metadata.labels[crossplane.io/claim-name] - toFieldPath: spec.writeConnectionSecretToRef.name - - type: ToCompositeFieldPath - fromFieldPath: metadata.labels[crossplane.io/claim-namespace] - toFieldPath: spec.writeConnectionSecretToRef.namespace - - type: ToCompositeFieldPath - fromFieldPath: metadata.labels[crossplane.io/claim-name] - toFieldPath: status.binding.name - connectionDetails: - - fromConnectionSecretKey: password - - name: release - base: - apiVersion: helm.crossplane.io/v1beta1 - kind: Release - spec: - forProvider: - namespace: upbound-system - chart: - name: postgresql - repository: https://charts.bitnami.com/bitnami - version: 12.1.2 - values: - architecture: standalone - global: - postgresql: - auth: - secretKeys: - adminPasswordKey: password - connectionDetails: - - name: type - value: postgresql - - name: provider - value: helm - - name: database - fromFieldPath: spec.forProvider.values.global.postgresql.auth.database - type: FromFieldPath - - name: username - fromFieldPath: spec.forProvider.values.global.postgresql.auth.username - type: FromFieldPath - - name: host - fromFieldPath: metadata.labels.address - type: FromFieldPath - - name: port - value: "5432" - patches: - - type: CombineFromComposite - toFieldPath: spec.forProvider.values.primary.initdb.args - policy: - fromFieldPath: Required - combine: - strategy: string - string: - fmt: -E %s --locale=%s - variables: - - fromFieldPath: spec.parameters.charset - - fromFieldPath: spec.parameters.collation - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.version - toFieldPath: spec.forProvider.values.image.tag - transforms: - - type: map - map: - "11": 11.18.0 - "12": 12.13.0 - "13": 13.9.0 - "14": 14.6.0 - "15": 15.1.0 - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.forProvider.values.global.postgresql.auth.existingSecret - transforms: - - type: string - string: - type: Format - fmt: '%s-postgresql-admin' - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.adminUsername - toFieldPath: spec.forProvider.values.global.postgresql.auth.username - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.storageClass - toFieldPath: spec.forProvider.values.primary.persistence.storageClass - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.database - toFieldPath: spec.forProvider.values.global.postgresql.auth.database - - type: FromCompositeFieldPath - fromFieldPath: status.address - toFieldPath: metadata.labels.address - - type: ToCompositeFieldPath - toFieldPath: status.location - fromFieldPath: spec.forProvider.namespace - - type: CombineToComposite - toFieldPath: status.address - policy: - fromFieldPath: Required - combine: - strategy: string - string: - fmt: '%s.%s.svc.cluster.local' - variables: - - fromFieldPath: metadata.name - - fromFieldPath: spec.forProvider.namespace - - type: ToCompositeFieldPath - fromFieldPath: spec.forProvider.values.image.tag - toFieldPath: status.version diff --git a/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml index e916a31..8dd73b8 100644 --- a/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml @@ -119,8 +119,6 @@ spec: namespace: #@ data.values.crossplane.namespace location: "" version: "" #! 11,12 and 13 are supported - writeConnectionSecretToRef: - namespace: #@ data.values.crossplane.namespace patches: - type: FromCompositeFieldPath fromFieldPath: spec.parameters.location @@ -136,14 +134,6 @@ spec: "13": "13" "14": "13" "15": "13" - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.writeConnectionSecretToRef.name - transforms: - - string: - fmt: '%s-postgresql-admin' - type: Format - type: string - type: FromCompositeFieldPath fromFieldPath: metadata.uid toFieldPath: spec.forProvider.administratorPasswordSecretRef.name diff --git a/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml index 0273276..48b4746 100644 --- a/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml @@ -57,10 +57,8 @@ spec: properties: startIpAddress: type: string - default: "0.0.0.0/0" endIpAddress: type: string - default: "0.0.0.0/0" required: - location required: From bdf7e7103fc8d6edba9904783c3b4fe30136e335 Mon Sep 17 00:00:00 2001 From: Joost van der Griendt Date: Tue, 6 Dec 2022 22:19:34 +0100 Subject: [PATCH 11/41] tweak the script to also support azure psql --- scripts/crossplane-e2e-multicloud-psql.sh | 21 +++++++++++-------- .../claim-azure-instance.sh | 9 ++++---- .../claim-helm-instance.sh | 4 ---- .../crossplane-e2e-multicloud-psql/cleanup.sh | 12 ++++++++++- .../crossplane-e2e-multicloud-psql/test.sh | 10 ++++++++- 5 files changed, 36 insertions(+), 20 deletions(-) diff --git a/scripts/crossplane-e2e-multicloud-psql.sh b/scripts/crossplane-e2e-multicloud-psql.sh index 995c2b2..d534aa8 100755 --- a/scripts/crossplane-e2e-multicloud-psql.sh +++ b/scripts/crossplane-e2e-multicloud-psql.sh @@ -7,7 +7,7 @@ AZURE_CREDS_SECRET_NAME=${AZURE_CREDS_SECRET_NAME:-"azure-secret"} UXP_VERSION=${UXP_VERSION:-"v1.10.1-up.1"} CONFIG_NAME=${CONFIG_NAME:-"trp-multicloud-psql"} CONFIG_IMAGE=${CONFIG_IMAGE:-"ghcr.io/vmware-tanzu-labs/trp-azure-psql"} -CONFIG_VERSION=${CONFIG_VERSION:-"0.0.0-0.0.11-14-g7e210b7"} +CONFIG_VERSION=${CONFIG_VERSION:-"0.0.1-rc-1"} CLAIM_NAME=${CLAIM_NAME:-"postgresql-0001"} TEST_APP_NAME=${TEST_APP_NAME:-"spring-boot-postgres"} STORAGE_CLASS=${STORAGE_CLASS:-"default"} @@ -30,20 +30,23 @@ echo "> Installing required providers" ./crossplane-e2e-install-k8s-provider.sh ./crossplane-e2e-install-tf-provider.sh - ./crossplane-e2e-multicloud-psql/install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} +./crossplane-e2e-multicloud-psql/claim-helm-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} +./crossplane-e2e-multicloud-psql/test.sh ${TEST_APP_NAME} -# ./crossplane-e2e-multicloud-psql/claim-helm-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} - -# ./crossplane-e2e-multicloud-psql/test.sh ${TEST_APP_NAME} +./crossplane-e2e-multicloud-psql/cleanup.sh -# kubectl delete postgresqlinstances ${CLAIM_NAME} || true -# kubectl delete deploy ${TEST_APP_NAME} || true +sleep 5 -# sleep 5 +./crossplane-e2e-install-azure-provider.sh ${CROSSPLANE_NAMESPACE} ${AZURE_CREDS_SECRET_NAME} +./crossplane-e2e-install-helm-provider.sh +./crossplane-e2e-install-k8s-provider.sh +./crossplane-e2e-install-tf-provider.sh +./crossplane-e2e-multicloud-psql/install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} ./crossplane-e2e-multicloud-psql/claim-azure-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} - ./crossplane-e2e-multicloud-psql/test.sh ${TEST_APP_NAME} +./crossplane-e2e-multicloud-psql/cleanup.sh + popd diff --git a/scripts/crossplane-e2e-multicloud-psql/claim-azure-instance.sh b/scripts/crossplane-e2e-multicloud-psql/claim-azure-instance.sh index 873ff26..e618ec8 100755 --- a/scripts/crossplane-e2e-multicloud-psql/claim-azure-instance.sh +++ b/scripts/crossplane-e2e-multicloud-psql/claim-azure-instance.sh @@ -22,20 +22,19 @@ spec: database: demo collation: en_GB.utf8 storageClass: ${STORAGE_CLASS} + firewallRule: + startIpAddress: "0.0.0.0" + endIpAddress: "255.255.255.255" EOF kubectl get providerconfig,xpostgresqlinstances,postgresqlinstances -echo ">> Installing Test Application" -kubectl apply -f https://raw.githubusercontent.com/joostvdg/spring-boot-postgres/main/kubernetes/deployment.yaml -kubectl get deployment - echo ">> Showing Secrets (1)" kubectl get secret -n ${CROSSPLANE_NAMESPACE} kubectl get secret echo ">> Waiting for Managed Resources To Get Ready" -kubectl wait --for=condition=ready postgresqlinstances.multi.ref.services.apps.tanzu.vmware.com/${CLAIM_NAME} --timeout=400s +kubectl wait --for=condition=ready postgresqlinstances.multi.ref.services.apps.tanzu.vmware.com/${CLAIM_NAME} --timeout=600s # We can also wait on the "release" echo ">> Showing Secrets (2)" diff --git a/scripts/crossplane-e2e-multicloud-psql/claim-helm-instance.sh b/scripts/crossplane-e2e-multicloud-psql/claim-helm-instance.sh index ad36506..2c37a45 100755 --- a/scripts/crossplane-e2e-multicloud-psql/claim-helm-instance.sh +++ b/scripts/crossplane-e2e-multicloud-psql/claim-helm-instance.sh @@ -27,10 +27,6 @@ EOF kubectl get providerconfig,xpostgresqlinstances,postgresqlinstances -echo ">> Installing Test Application" -kubectl apply -f https://raw.githubusercontent.com/joostvdg/spring-boot-postgres/main/kubernetes/deployment.yaml -kubectl get deployment - echo ">> Showing Secrets (1)" kubectl get secret -n ${CROSSPLANE_NAMESPACE} kubectl get secret diff --git a/scripts/crossplane-e2e-multicloud-psql/cleanup.sh b/scripts/crossplane-e2e-multicloud-psql/cleanup.sh index 1811256..650298d 100755 --- a/scripts/crossplane-e2e-multicloud-psql/cleanup.sh +++ b/scripts/crossplane-e2e-multicloud-psql/cleanup.sh @@ -11,9 +11,19 @@ kubectl delete postgresqlinstances ${CLAIM_NAME} || true kubectl delete deploy ${TEST_APP_NAME} || true kubectl delete configuration ${CONFIG_NAME} || true +kubectl delete flexibleserverconfigurations.dbforpostgresql.azure.upbound.io -l crossplane.io/claim-name=${CLAIM_NAME} --force --grace-period=0 || true +kubectl delete flexibleserverdatabases.dbforpostgresql.azure.upbound.io -l crossplane.io/claim-name=${CLAIM_NAME} --force --grace-period=0 || true +kubectl delete flexibleserverfirewallrules.dbforpostgresql.azure.upbound.io -l crossplane.io/claim-name=${CLAIM_NAME} --force --grace-period=0 || true + +FLEXIBLE_SERVER_NAME=$(kubectl get flexibleserver.dbforpostgresql.azure.upbound.io -l crossplane.io/claim-name=${CLAIM_NAME} -o name) +kubectl patch ${FLEXIBLE_SERVER_NAME} -p '{"metadata":{"finalizers":null}}' --type=merge || true +kubectl delete flexibleserver.dbforpostgresql.azure.upbound.io -l crossplane.io/claim-name=${CLAIM_NAME} --force --grace-period=0 || true + + kubectl delete providerconfigs.helm.crossplane.io default || true kubectl delete providerconfigs.kubernetes.crossplane.io default || true kubectl delete providers.pkg.crossplane.io crossplane-contrib-provider-helm || true kubectl delete providers.pkg.crossplane.io crossplane-contrib-provider-kubernetes || true -kubectl delete providers.pkg.crossplane.io crossplane-contrib-provider-terraform || true \ No newline at end of file +kubectl delete providers.pkg.crossplane.io crossplane-contrib-provider-terraform || true + diff --git a/scripts/crossplane-e2e-multicloud-psql/test.sh b/scripts/crossplane-e2e-multicloud-psql/test.sh index b932553..6f31f36 100755 --- a/scripts/crossplane-e2e-multicloud-psql/test.sh +++ b/scripts/crossplane-e2e-multicloud-psql/test.sh @@ -2,9 +2,17 @@ TEST_APP_NAME=$1 +echo ">> Installing Test Application" +kubectl apply -f https://raw.githubusercontent.com/joostvdg/spring-boot-postgres/main/kubernetes/deployment.yaml +kubectl get deployment + + echo ">> Waiting on Test Application: ${TEST_APP_NAME}" kubectl get pod -kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=${TEST_APP_NAME} --timeout=60s +kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=${TEST_APP_NAME} --timeout=300s + +kubectl describe pod -l app.kubernetes.io/name=${TEST_APP_NAME} +sleep 10 echo ">> Starting Port Forward" kubectl port-forward deployment/${TEST_APP_NAME} 8080 & From 32140ddfcf7e7d177dda88efca32eb2850c6d181 Mon Sep 17 00:00:00 2001 From: Joost van der Griendt Date: Wed, 14 Dec 2022 16:00:10 +0100 Subject: [PATCH 12/41] tf provider has been replaced with the official one --- scripts/crossplane-e2e-install-tf-provider.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/crossplane-e2e-install-tf-provider.sh b/scripts/crossplane-e2e-install-tf-provider.sh index 341f735..aaa4942 100755 --- a/scripts/crossplane-e2e-install-tf-provider.sh +++ b/scripts/crossplane-e2e-install-tf-provider.sh @@ -1,5 +1,13 @@ -up controlplane provider install xpkg.upbound.io/crossplane-contrib/provider-terraform:v0.4.0 || true +cat < Date: Wed, 14 Dec 2022 16:00:47 +0100 Subject: [PATCH 13/41] add safeguard against script failing --- scripts/crossplane-e2e-multicloud-psql.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/crossplane-e2e-multicloud-psql.sh b/scripts/crossplane-e2e-multicloud-psql.sh index d534aa8..3c6bb3f 100755 --- a/scripts/crossplane-e2e-multicloud-psql.sh +++ b/scripts/crossplane-e2e-multicloud-psql.sh @@ -25,6 +25,8 @@ pushd $(dirname $0) echo "> Installing required providers" +trap $(dirname $0)/crossplane-e2e-multicloud-psql/cleanup.sh EXIT + ./crossplane-e2e-install-azure-provider.sh ${CROSSPLANE_NAMESPACE} ${AZURE_CREDS_SECRET_NAME} ./crossplane-e2e-install-helm-provider.sh ./crossplane-e2e-install-k8s-provider.sh From b09da4c3442a4cde55ceee87101f69e2ca2fa007 Mon Sep 17 00:00:00 2001 From: Joost van der Griendt Date: Wed, 14 Dec 2022 16:01:51 +0100 Subject: [PATCH 14/41] add aws rds composition --- .../psql/wip/aws-composition.ytt.yml | 111 ----- .../psql/ytt/aws-composition.ytt.yml | 415 ++++++++++++++++++ .../crossplane/psql/ytt/definition.ytt.yml | 21 + .../crossplane/psql/ytt/schema.ytt.yml | 6 + 4 files changed, 442 insertions(+), 111 deletions(-) delete mode 100644 packages/multicloud/crossplane/psql/wip/aws-composition.ytt.yml create mode 100644 packages/multicloud/crossplane/psql/ytt/aws-composition.ytt.yml diff --git a/packages/multicloud/crossplane/psql/wip/aws-composition.ytt.yml b/packages/multicloud/crossplane/psql/wip/aws-composition.ytt.yml deleted file mode 100644 index def88fb..0000000 --- a/packages/multicloud/crossplane/psql/wip/aws-composition.ytt.yml +++ /dev/null @@ -1,111 +0,0 @@ -#@ load("@ytt:data", "data") ---- -apiVersion: apiextensions.crossplane.io/v1 -kind: Composition -metadata: - name: #@ (data.values.xrd.claimNames.kind).lower() - labels: - crossplane.io/xrd: #@ data.values.xrd.names.plural + "." + data.values.xrd.group - provider: #@ data.values.provider.name - database: #@ data.values.cloudServiceBindingType -spec: - publishConnectionDetailsWithStoreConfigRef: - name: #@ data.values.storeConfig.name - compositeTypeRef: - apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version - kind: #@ data.values.xrd.names.kind - resources: - - base: - apiVersion: rds.aws.upbound.io/v1beta1 - kind: Instance - metadata: - annotations: - upjet.upbound.io/manual-intervention: This resource has a password secret reference. - name: example-dbinstance - spec: - forProvider: - allocatedStorage: 20 - autoMinorVersionUpgrade: true - backupRetentionPeriod: 14 - backupWindow: 09:46-10:16 - engine: postgres - engineVersion: "13.7" - instanceClass: db.t3.micro - maintenanceWindow: Mon:00:00-Mon:03:00 - name: example - passwordSecretRef: #! needs password generated by secret gen as option?! - key: password - name: example-dbinstance - namespace: upbound-system - publiclyAccessible: false - region: us-west-1 - skipFinalSnapshot: true - storageEncrypted: false - storageType: gp2 #! One of "standard" (magnetic), "gp2" (general purpose SSD), or "io1" (provisioned IOPS SSD). The default is "io1" if iops is specified, "gp2" if not. - username: adminuser - optionGroupName: abc #! needs reference/api resouce - parameterGroupName: abc #! needs reference/api resouce - vpcSecurityGroupIdSelector: #! needs reference/api resouce - matchControllerRef: true #! needs reference/api resouce - dbSubnetGroupNameSelector: - matchControllerRef: true #! needs reference/api resouce - providerConfigRef: - name: #@ data.values.provider.configRef - writeConnectionSecretToRef: - namespace: #@ data.values.crossplane.namespace - #! if specified, else it goes into "default VPC" - - base: - apiVersion: rds.aws.upbound.io/v1beta1 - kind: SubnetGroup - metadata: - name: example-subnetgroup - spec: - forProvider: - region: us-west-1 - subnetIdRefs: - - name: sample-db-subnet1 - - name: sample-db-subnet2 - tags: - Name: My DB subnet group - - #! OLD, needs to be replaced - - base: - apiVersion: database.aws.crossplane.io/v1beta1 - kind: RDSInstance - spec: - forProvider: - dbInstanceClass: db.t2.micro - engine: postgres - dbName: postgres - engineVersion: "12" - masterUsername: masteruser - publiclyAccessible: true - region: us-east-1 - skipFinalSnapshotBeforeDeletion: true - writeConnectionSecretToRef: - namespace: #@ data.values.crossplane.namespace - connectionDetails: - - name: type - value: postgresql - - name: provider - value: aws - - name: database - value: postgres - - fromConnectionSecretKey: username - - fromConnectionSecretKey: password - - name: host - fromConnectionSecretKey: endpoint - - fromConnectionSecretKey: port - name: rdsinstance - patches: - - fromFieldPath: metadata.uid - toFieldPath: spec.writeConnectionSecretToRef.name - transforms: - - string: - fmt: '%s-postgresql' - type: Format - type: string - type: FromCompositeFieldPath - - fromFieldPath: spec.parameters.storageGB - toFieldPath: spec.forProvider.allocatedStorage - type: FromCompositeFieldPath \ No newline at end of file diff --git a/packages/multicloud/crossplane/psql/ytt/aws-composition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/aws-composition.ytt.yml new file mode 100644 index 0000000..a6edcc2 --- /dev/null +++ b/packages/multicloud/crossplane/psql/ytt/aws-composition.ytt.yml @@ -0,0 +1,415 @@ +#@ load("@ytt:data", "data") +--- +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: #@ data.values.providers.aws.name + "-" + data.values.cloudServiceBindingType + labels: + crossplane.io/xrd: #@ data.values.xrd.names.plural + "." + data.values.xrd.group + provider: #@ data.values.providers.aws.name + database: #@ data.values.cloudServiceBindingType +spec: + writeConnectionSecretsToNamespace: #@ data.values.crossplane.namespace + compositeTypeRef: + apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version + kind: #@ data.values.xrd.names.kind + resources: + - name: tf-providerconfig + base: + apiVersion: kubernetes.crossplane.io/v1alpha1 + kind: Object + spec: + forProvider: + manifest: + apiVersion: tf.upbound.io/v1beta1 + kind: ProviderConfig + spec: + credentials: [] + #@yaml/text-templated-strings + configuration: | + terraform { + required_providers { + random = { + source = "hashicorp/random" + version = "3.4.3" + } + } + + backend "kubernetes" { + secret_suffix = "providerconfig-default" + namespace = "(@= data.values.crossplane.namespace @)" + in_cluster_config = true + } + } + provider "random" {} + patches: + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.forProvider.manifest.metadata.name + - name: password + base: + apiVersion: tf.upbound.io/v1beta1 + kind: Workspace + spec: + forProvider: + module: | + resource "random_password" "password" { + length = 64 + special = false + } + + output "password" { + value = random_password.password.result + sensitive = true + } + source: Inline + writeConnectionSecretToRef: + namespace: #@ data.values.crossplane.namespace + patches: + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.writeConnectionSecretToRef.name + transforms: + - type: string + string: + type: Format + fmt: '%s-postgresql-admin' + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.providerConfigRef.name + - type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-name] + toFieldPath: spec.writeConnectionSecretToRef.name + - type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-namespace] + toFieldPath: spec.writeConnectionSecretToRef.namespace + - type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-name] + toFieldPath: status.binding.name + connectionDetails: + - fromConnectionSecretKey: password + - name: rdsinstance + base: + apiVersion: rds.aws.upbound.io/v1beta1 + kind: Instance + spec: + forProvider: + engine: postgres + instanceClass: db.t3.micro + passwordSecretRef: #! needs password generated by secret gen as option?! + key: password + namespace: #@ data.values.crossplane.namespace + publiclyAccessible: true #! set this to false for anything other than temporary demos + skipFinalSnapshot: true + storageEncrypted: false + allocatedStorage: 10 + #! storageType: gp2 #! One of "standard" (magnetic), "gp2" (general purpose SSD), or "io1" (provisioned IOPS SSD). The default is "io1" if iops is specified, "gp2" if not. + #!optionGroupName: abc #! needs reference/api resouce + #!parameterGroupName: abc #! needs reference/api resouce + vpcSecurityGroupIdSelector: + matchControllerRef: true + dbSubnetGroupNameSelector: + matchControllerRef: true + providerConfigRef: + name: #@ data.values.providers.aws.configRef + patches: + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.forProvider.passwordSecretRef.name + transforms: + - type: string + string: + type: Format + fmt: '%s-postgresql-admin' + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.version + toFieldPath: spec.forProvider.engineVersion + transforms: + - type: map + map: + "11": "11.17" + "12": "12.12" + "13": "13.8" + "14": "14.5" + "15": "14.5" + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.adminUsername + toFieldPath: spec.forProvider.username + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.database + toFieldPath: spec.forProvider.dbName + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.storageClass + toFieldPath: spec.forProvider.storageType + #! not allowed when using 11, 12, not sure about others + #! - type: FromCompositeFieldPath + #! fromFieldPath: spec.parameters.charset + #! toFieldPath: spec.forProvider.characterSetName + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.collation + toFieldPath: spec.forProvider.collation + + connectionDetails: + - name: type + value: postgresql + - name: provider + value: aws + - name: port + value: "5432" + - name: database + fromFieldPath: spec.forProvider.dbName + type: FromFieldPath + - name: username + fromFieldPath: spec.forProvider.username + type: FromFieldPath + - name: host + fromFieldPath: status.atProvider.address + type: FromFieldPath + + + - name: securitygroup + base: + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: SecurityGroup + patches: + - type: FromCompositeFieldPath + fromFieldPath: metadata.name + toFieldPath: spec.forProvider.name + - type: FromCompositeFieldPath + fromFieldPath: metadata.name + toFieldPath: spec.forProvider.description + transforms: + - string: + fmt: 'Traffic to RDS instance %s' + type: Format + type: string + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.vpcId + toFieldPath: spec.forProvider.vpcId + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region + + - name: securitygrouprule + base: + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: SecurityGroupRule + spec: + forProvider: + fromPort: 5432 + toPort: 5432 + protocol: tcp + securityGroupIdSelector: + matchControllerRef: true + type: ingress + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.cidrBlocks + toFieldPath: spec.forProvider.cidrBlocks + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region + - type: FromCompositeFieldPath + fromFieldPath: metadata.labels.port + toFieldPath: spec.forProvider.fromPort + transforms: + - type: convert + convert: + toType: int + - type: FromCompositeFieldPath + fromFieldPath: metadata.labels.port + toFieldPath: spec.forProvider.toPort + transforms: + - type: convert + convert: + toType: int + + - name: subnetgroup + base: + apiVersion: rds.aws.upbound.io/v1beta1 + kind: SubnetGroup + spec: + forProvider: + subnetIdSelector: + matchControllerRef: true + tags: + Name: subnet group for RDS via crossplane + patches: + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: metadata.name + transforms: + - type: string + string: + type: Format + fmt: 'rds-sg-%s' + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region + + - name: subneta + base: + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: Subnet + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.vpcId + toFieldPath: spec.forProvider.vpcId + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.subnetACidrBlock + toFieldPath: spec.forProvider.cidrBlock + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: metadata.name + transforms: + - type: string + string: + type: Format + fmt: 'subnet-a-%s' + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.availabilityZone + transforms: + - type: string + string: + type: Format + fmt: '%sa' + - name: subnetb + base: + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: Subnet + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.vpcId + toFieldPath: spec.forProvider.vpcId + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.subnetBCidrBlock + toFieldPath: spec.forProvider.cidrBlock + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: metadata.name + transforms: + - type: string + string: + type: Format + fmt: 'subnet-b-%s' + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.availabilityZone + transforms: + - type: string + string: + type: Format + fmt: '%sb' + - name: routetable + base: + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: RouteTable + metadata: + name: table + spec: + forProvider: + tags: + Name: RDS-via-Crossplane + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.vpcId + toFieldPath: spec.forProvider.vpcId + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region + - name: routetableassoA + base: + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: RouteTableAssociation + metadata: + name: rta + spec: + forProvider: + routeTableIdSelector: + matchControllerRef: true + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: metadata.name + transforms: + - type: string + string: + type: Format + fmt: 'subnet-a-%s' + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.forProvider.subnetIdRef.name + transforms: + - type: string + string: + type: Format + fmt: 'subnet-a-%s' + - name: routetableassoB + base: + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: RouteTableAssociation + metadata: + name: rta + spec: + forProvider: + routeTableIdSelector: + matchControllerRef: true + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: metadata.name + transforms: + - type: string + string: + type: Format + fmt: 'subnet-b-%s' + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.forProvider.subnetIdRef.name + transforms: + - type: string + string: + type: Format + fmt: 'subnet-b-%s' + - name: route + base: + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: Route + metadata: + name: route + spec: + forProvider: + destinationCidrBlock: 0.0.0.0/0 + routeTableIdSelector: + matchControllerRef: true + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.gatewayId + toFieldPath: spec.forProvider.gatewayId + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: metadata.name + transforms: + - type: string + string: + type: Format + fmt: 'route-%s' \ No newline at end of file diff --git a/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml index 48b4746..4c294f8 100644 --- a/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml @@ -49,6 +49,27 @@ spec: collation: type: string default: en_US.utf8 + gatewayId: + type: string + description: (only public AWS) + subnetACidrBlock: + type: string + description: (only AWS) + subnetBCidrBlock: + type: string + description: (only AWS) + cidrBlocks: + type: array + description: (only AWS) cidr blocks for AWS security group + items: + type: string + pattern: '^(1?\d{1,2}|2[01234]\d|25[012345])(\.(1?\d{1,2}|2[01234]\d|25[012345])){3}\/(\d|[12]\d|3[012])$' + dbSubnetGroupName: + type: string + description: (only AWS) name for the AWS subnet group + vpcId: + type: string + description: (only AWS) id of the AWS vpc charset: type: string default: utf8 diff --git a/packages/multicloud/crossplane/psql/ytt/schema.ytt.yml b/packages/multicloud/crossplane/psql/ytt/schema.ytt.yml index 1e07cc2..2c07057 100644 --- a/packages/multicloud/crossplane/psql/ytt/schema.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/schema.ytt.yml @@ -26,6 +26,12 @@ providers: image: xpkg.upbound.io/crossplane-contrib/provider-helm version: ">=v0.12.0" + aws: + name: aws + image: xpkg.upbound.io/upbound/provider-aws + version: ">=v0.22.0" + configRef: default + crossplane: #@schema/title "CrossplaneNamespace" #@schema/desc "The namespace where crossplane controller is installed" From a9ed4ca0ec5962d9d17ba995ff92686eaadaf44e Mon Sep 17 00:00:00 2001 From: Joost van der Griendt Date: Fri, 16 Dec 2022 10:40:38 +0100 Subject: [PATCH 15/41] implemented aws rds for public and private access --- .../psql/claim-examples/aws-private.yaml | 22 + .../psql/claim-examples/aws-public.yaml | 26 ++ .../psql/claim-examples/azure-psql.yaml | 19 + .../psql/ytt/aws-composition-private.ytt.yml | 25 ++ .../psql/ytt/aws-composition-public.ytt.yml | 41 ++ .../psql/ytt/aws-composition.lib.yml | 366 +++++++++++++++ .../psql/ytt/aws-composition.ytt.yml | 415 ------------------ .../crossplane/psql/ytt/definition.ytt.yml | 72 +-- 8 files changed, 545 insertions(+), 441 deletions(-) create mode 100644 packages/multicloud/crossplane/psql/claim-examples/aws-private.yaml create mode 100644 packages/multicloud/crossplane/psql/claim-examples/aws-public.yaml create mode 100644 packages/multicloud/crossplane/psql/claim-examples/azure-psql.yaml create mode 100644 packages/multicloud/crossplane/psql/ytt/aws-composition-private.ytt.yml create mode 100644 packages/multicloud/crossplane/psql/ytt/aws-composition-public.ytt.yml create mode 100644 packages/multicloud/crossplane/psql/ytt/aws-composition.lib.yml delete mode 100644 packages/multicloud/crossplane/psql/ytt/aws-composition.ytt.yml diff --git a/packages/multicloud/crossplane/psql/claim-examples/aws-private.yaml b/packages/multicloud/crossplane/psql/claim-examples/aws-private.yaml new file mode 100644 index 0000000..18f1e17 --- /dev/null +++ b/packages/multicloud/crossplane/psql/claim-examples/aws-private.yaml @@ -0,0 +1,22 @@ +apiVersion: multi.ref.services.apps.tanzu.vmware.com/v1alpha1 +kind: PostgreSQLInstance +metadata: + name: aws-psql-private-01 + labels: + services.apps.tanzu.vmware.com/claimable: "true" +spec: + compositionSelector: + matchLabels: + provider: aws + connectivity: "private" + parameters: + location: eu-central-1 + version: "12" + database: demo + collation: en_GB.utf8 + storageClass: gp2 + aws: + vpcId: vpc-0157e2424963dcd93 + dbSubnetGroupName: trp-testing + cidrBlocks: + - 0.0.0.0/0 \ No newline at end of file diff --git a/packages/multicloud/crossplane/psql/claim-examples/aws-public.yaml b/packages/multicloud/crossplane/psql/claim-examples/aws-public.yaml new file mode 100644 index 0000000..e1b7f62 --- /dev/null +++ b/packages/multicloud/crossplane/psql/claim-examples/aws-public.yaml @@ -0,0 +1,26 @@ +apiVersion: multi.ref.services.apps.tanzu.vmware.com/v1alpha1 +kind: PostgreSQLInstance +metadata: + name: aws-psql-public-01 + labels: + services.apps.tanzu.vmware.com/claimable: "true" +spec: + compositionSelector: + matchLabels: + provider: aws + connectivity: "public" + parameters: + location: eu-central-1 + version: "12" + database: demo + collation: en_GB.utf8 + storageClass: gp2 + aws: + vpcId: vpc-0157e2424963dcd93 + dbSubnetGroupName: trp-testing + cidrBlocks: + - 0.0.0.0/0 + public: + gatewayId: igw-03e173382a35db8ab + subnetACidrBlock: "10.100.255.0/25" + subnetBCidrBlock: "10.100.255.128/25" diff --git a/packages/multicloud/crossplane/psql/claim-examples/azure-psql.yaml b/packages/multicloud/crossplane/psql/claim-examples/azure-psql.yaml new file mode 100644 index 0000000..6c28f05 --- /dev/null +++ b/packages/multicloud/crossplane/psql/claim-examples/azure-psql.yaml @@ -0,0 +1,19 @@ +apiVersion: multi.ref.services.apps.tanzu.vmware.com/v1alpha1 +kind: PostgreSQLInstance +metadata: + name: my-azure-psql + labels: + services.apps.tanzu.vmware.com/claimable: "true" +spec: + compositionSelector: + matchLabels: + provider: azure + parameters: + location: "West Europe" + version: "12" + database: demo + collation: en_GB.utf8 + storageClass: hostpath + firewallRule: + startIpAddress: "0.0.0.0" + endIpAddress: "255.255.255.255" \ No newline at end of file diff --git a/packages/multicloud/crossplane/psql/ytt/aws-composition-private.ytt.yml b/packages/multicloud/crossplane/psql/ytt/aws-composition-private.ytt.yml new file mode 100644 index 0000000..7dbdfe0 --- /dev/null +++ b/packages/multicloud/crossplane/psql/ytt/aws-composition-private.ytt.yml @@ -0,0 +1,25 @@ +#@ load("@ytt:data", "data") +--- +#@ load("aws-composition.lib.yml", "tfProviderConfig", "tfWorkspace", "rdsInstance", "securityGroup", "securityGroupRule") + +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: #@ data.values.providers.aws.name + "-" + data.values.cloudServiceBindingType + "-private" + labels: + crossplane.io/xrd: #@ data.values.xrd.names.plural + "." + data.values.xrd.group + provider: #@ data.values.providers.aws.name + database: #@ data.values.cloudServiceBindingType + connectivity: "private" +spec: + writeConnectionSecretsToNamespace: #@ data.values.crossplane.namespace + compositeTypeRef: + apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version + kind: #@ data.values.xrd.names.kind + resources: + - #@ tfProviderConfig(data.values.crossplane.namespace) + - #@ tfWorkspace(data.values.crossplane.namespace) + - #@ rdsInstance(data.values.crossplane.namespace, data.values.providers.aws.configRef, False) + - #@ securityGroup() + - #@ securityGroupRule() + diff --git a/packages/multicloud/crossplane/psql/ytt/aws-composition-public.ytt.yml b/packages/multicloud/crossplane/psql/ytt/aws-composition-public.ytt.yml new file mode 100644 index 0000000..93fc3c3 --- /dev/null +++ b/packages/multicloud/crossplane/psql/ytt/aws-composition-public.ytt.yml @@ -0,0 +1,41 @@ +#@ load("@ytt:data", "data") +--- +#@ load("aws-composition.lib.yml", "tfProviderConfig", "tfWorkspace", "rdsInstance", "securityGroup", "securityGroupRule", "subnet", "routeTableAssociation", "routeTable", "route", "subnetGroup") + +#@ subnetASuffix = '-a' +#@ subnetAFormat = 'subnet-a-%s' +#@ availabilityZoneAFormat = '%sa' +#@ cidrBlockFieldA = 'spec.parameters.aws.public.subnetACidrBlock' + +#@ subnetBSuffix = '-b' +#@ subnetBFormat = 'subnet-b-%s' +#@ availabilityZoneBFormat = '%sb' +#@ cidrBlockFieldB = 'spec.parameters.aws.public.subnetBCidrBlock' + +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: #@ data.values.providers.aws.name + "-" + data.values.cloudServiceBindingType + "-public" + labels: + crossplane.io/xrd: #@ data.values.xrd.names.plural + "." + data.values.xrd.group + provider: #@ data.values.providers.aws.name + database: #@ data.values.cloudServiceBindingType + connectivity: "public" +spec: + writeConnectionSecretsToNamespace: #@ data.values.crossplane.namespace + compositeTypeRef: + apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version + kind: #@ data.values.xrd.names.kind + resources: + - #@ tfProviderConfig(data.values.crossplane.namespace) + - #@ tfWorkspace(data.values.crossplane.namespace) + - #@ rdsInstance(data.values.crossplane.namespace, data.values.providers.aws.configRef, True) + - #@ securityGroup() + - #@ securityGroupRule() + - #@ routeTable() + - #@ subnetGroup() + - #@ subnet(subnetASuffix, subnetAFormat, availabilityZoneAFormat, cidrBlockFieldA) + - #@ routeTableAssociation(subnetASuffix, subnetAFormat) + - #@ subnet(subnetBSuffix, subnetBFormat, availabilityZoneBFormat, cidrBlockFieldB) + - #@ routeTableAssociation(subnetBSuffix, subnetBFormat) + - #@ route('route-%s') \ No newline at end of file diff --git a/packages/multicloud/crossplane/psql/ytt/aws-composition.lib.yml b/packages/multicloud/crossplane/psql/ytt/aws-composition.lib.yml new file mode 100644 index 0000000..db3370d --- /dev/null +++ b/packages/multicloud/crossplane/psql/ytt/aws-composition.lib.yml @@ -0,0 +1,366 @@ +#@ def subnetGroupNameMatcher(): + +#@ end + +#@ def tfProviderConfig(crossplaneNamespace): +name: tf-providerconfig +base: + apiVersion: kubernetes.crossplane.io/v1alpha1 + kind: Object + spec: + forProvider: + manifest: + apiVersion: tf.upbound.io/v1beta1 + kind: ProviderConfig + spec: + credentials: [] + #@yaml/text-templated-strings + configuration: | + terraform { + required_providers { + random = { + source = "hashicorp/random" + version = "3.4.3" + } + } + + backend "kubernetes" { + secret_suffix = "providerconfig-default" + namespace = "(@= crossplaneNamespace @)" + in_cluster_config = true + } + } + provider "random" {} +patches: +- type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.forProvider.manifest.metadata.name +#@ end + + +#@ def tfWorkspace(crossplaneNamespace): +name: password +base: + apiVersion: tf.upbound.io/v1beta1 + kind: Workspace + spec: + forProvider: + module: | + resource "random_password" "password" { + length = 64 + special = false + } + + output "password" { + value = random_password.password.result + sensitive = true + } + source: Inline + writeConnectionSecretToRef: + namespace: #@ crossplaneNamespace +patches: +- type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.writeConnectionSecretToRef.name + transforms: + - type: string + string: + type: Format + fmt: '%s-postgresql-admin' +- type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.providerConfigRef.name +- type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-name] + toFieldPath: spec.writeConnectionSecretToRef.name +- type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-namespace] + toFieldPath: spec.writeConnectionSecretToRef.namespace +- type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-name] + toFieldPath: status.binding.name +connectionDetails: +- fromConnectionSecretKey: password +#@ end + +#@ def rdsInstance(crossplaneNamespace, providerConfigRef, publiclyAccessible): +name: rdsinstance +base: + apiVersion: rds.aws.upbound.io/v1beta1 + kind: Instance + spec: + forProvider: + engine: postgres + instanceClass: db.t3.micro + passwordSecretRef: + key: password + namespace: #@ crossplaneNamespace + publiclyAccessible: #@ publiclyAccessible + skipFinalSnapshot: true + storageEncrypted: false + allocatedStorage: 10 + vpcSecurityGroupIdSelector: + matchControllerRef: true + #@ if/end publiclyAccessible: + dbSubnetGroupNameSelector: + matchControllerRef: true + providerConfigRef: + name: #@ providerConfigRef +patches: +#@ if/end not publiclyAccessible: +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.aws.dbSubnetGroupName + toFieldPath: spec.forProvider.dbSubnetGroupName +- type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.forProvider.passwordSecretRef.name + transforms: + - type: string + string: + type: Format + fmt: '%s-postgresql-admin' +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.version + toFieldPath: spec.forProvider.engineVersion + transforms: + - type: map + map: + "11": "11.17" + "12": "12.12" + "13": "13.8" + "14": "14.5" + "15": "14.5" +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.adminUsername + toFieldPath: spec.forProvider.username +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.database + toFieldPath: spec.forProvider.dbName +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.storageClass + toFieldPath: spec.forProvider.storageType +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.collation + toFieldPath: spec.forProvider.collation +connectionDetails: +- name: type + value: postgresql +- name: provider + value: aws +- name: port + value: "5432" +- name: database + fromFieldPath: spec.forProvider.dbName + type: FromFieldPath +- name: username + fromFieldPath: spec.forProvider.username + type: FromFieldPath +- name: host + fromFieldPath: status.atProvider.address + type: FromFieldPath +- name: securitygroup +#@ end + +#@ def securityGroup(): +name: securitygroup +base: + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: SecurityGroup +patches: +- type: FromCompositeFieldPath + fromFieldPath: metadata.name + toFieldPath: spec.forProvider.name +- type: FromCompositeFieldPath + fromFieldPath: metadata.name + toFieldPath: spec.forProvider.description + transforms: + - string: + fmt: 'Traffic to RDS instance %s' + type: Format + type: string +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.aws.vpcId + toFieldPath: spec.forProvider.vpcId +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region +#@ end + +#@ def securityGroupRule(): +name: securitygrouprule +base: + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: SecurityGroupRule + spec: + forProvider: + fromPort: 5432 + toPort: 5432 + protocol: tcp + securityGroupIdSelector: + matchControllerRef: true + type: ingress +patches: +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.aws.cidrBlocks + toFieldPath: spec.forProvider.cidrBlocks +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region +#@ end + +#@ def subnetGroup(): +name: subnetgroup +base: + apiVersion: rds.aws.upbound.io/v1beta1 + kind: SubnetGroup + spec: + forProvider: + subnetIdSelector: + matchControllerRef: true + tags: + Name: subnet group for RDS via crossplane +patches: +- type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: metadata.name + transforms: + - type: string + string: + type: Format + fmt: 'rds-sg-%s' +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region +#@ end + +#@ def subnet(nameSuffix, nameFormat, availabilityZoneFormat, cidrBlockField): +name: #@ "subnet" + nameSuffix +base: + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: Subnet +patches: +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.aws.vpcId + toFieldPath: spec.forProvider.vpcId +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region +- type: FromCompositeFieldPath + fromFieldPath: #@ cidrBlockField + toFieldPath: spec.forProvider.cidrBlock +- type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: metadata.name + transforms: + - type: string + string: + type: Format + fmt: #@ nameFormat +- type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.forProfider.tags.Name + transforms: + - type: string + string: + type: Format + fmt: #@ nameFormat +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.availabilityZone + transforms: + - type: string + string: + type: Format + fmt: #@ availabilityZoneFormat +#@ end + +#@ def routeTable(): +name: routetable +base: + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: RouteTable + metadata: + name: table + spec: + forProvider: + tags: + Name: RDS-via-Crossplane +patches: +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.aws.vpcId + toFieldPath: spec.forProvider.vpcId +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region +- type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: metadata.name + transforms: + - type: string + string: + type: Format + fmt: 'rds-%s' +#@ end + +#@ def routeTableAssociation(nameSuffix, subnetNameFormat): +name: #@ "routeTableAssociation" + nameSuffix +base: + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: RouteTableAssociation + spec: + forProvider: + routeTableIdSelector: + matchControllerRef: true +patches: +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region +- type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: metadata.name + transforms: + - type: string + string: + type: Format + fmt: #@ subnetNameFormat +- type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.forProvider.subnetIdRef.name + transforms: + - type: string + string: + type: Format + fmt: #@ subnetNameFormat +#@ end + +#@ def route(routeFormat): +name: route +base: + apiVersion: ec2.aws.upbound.io/v1beta1 + kind: Route + metadata: + name: route + spec: + forProvider: + destinationCidrBlock: 0.0.0.0/0 + routeTableIdSelector: + matchControllerRef: true +patches: +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.region +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.aws.public.gatewayId + toFieldPath: spec.forProvider.gatewayId +- type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: metadata.name + transforms: + - type: string + string: + type: Format + fmt: #@ routeFormat +#@ end \ No newline at end of file diff --git a/packages/multicloud/crossplane/psql/ytt/aws-composition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/aws-composition.ytt.yml deleted file mode 100644 index a6edcc2..0000000 --- a/packages/multicloud/crossplane/psql/ytt/aws-composition.ytt.yml +++ /dev/null @@ -1,415 +0,0 @@ -#@ load("@ytt:data", "data") ---- -apiVersion: apiextensions.crossplane.io/v1 -kind: Composition -metadata: - name: #@ data.values.providers.aws.name + "-" + data.values.cloudServiceBindingType - labels: - crossplane.io/xrd: #@ data.values.xrd.names.plural + "." + data.values.xrd.group - provider: #@ data.values.providers.aws.name - database: #@ data.values.cloudServiceBindingType -spec: - writeConnectionSecretsToNamespace: #@ data.values.crossplane.namespace - compositeTypeRef: - apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version - kind: #@ data.values.xrd.names.kind - resources: - - name: tf-providerconfig - base: - apiVersion: kubernetes.crossplane.io/v1alpha1 - kind: Object - spec: - forProvider: - manifest: - apiVersion: tf.upbound.io/v1beta1 - kind: ProviderConfig - spec: - credentials: [] - #@yaml/text-templated-strings - configuration: | - terraform { - required_providers { - random = { - source = "hashicorp/random" - version = "3.4.3" - } - } - - backend "kubernetes" { - secret_suffix = "providerconfig-default" - namespace = "(@= data.values.crossplane.namespace @)" - in_cluster_config = true - } - } - provider "random" {} - patches: - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.forProvider.manifest.metadata.name - - name: password - base: - apiVersion: tf.upbound.io/v1beta1 - kind: Workspace - spec: - forProvider: - module: | - resource "random_password" "password" { - length = 64 - special = false - } - - output "password" { - value = random_password.password.result - sensitive = true - } - source: Inline - writeConnectionSecretToRef: - namespace: #@ data.values.crossplane.namespace - patches: - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.writeConnectionSecretToRef.name - transforms: - - type: string - string: - type: Format - fmt: '%s-postgresql-admin' - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.providerConfigRef.name - - type: ToCompositeFieldPath - fromFieldPath: metadata.labels[crossplane.io/claim-name] - toFieldPath: spec.writeConnectionSecretToRef.name - - type: ToCompositeFieldPath - fromFieldPath: metadata.labels[crossplane.io/claim-namespace] - toFieldPath: spec.writeConnectionSecretToRef.namespace - - type: ToCompositeFieldPath - fromFieldPath: metadata.labels[crossplane.io/claim-name] - toFieldPath: status.binding.name - connectionDetails: - - fromConnectionSecretKey: password - - name: rdsinstance - base: - apiVersion: rds.aws.upbound.io/v1beta1 - kind: Instance - spec: - forProvider: - engine: postgres - instanceClass: db.t3.micro - passwordSecretRef: #! needs password generated by secret gen as option?! - key: password - namespace: #@ data.values.crossplane.namespace - publiclyAccessible: true #! set this to false for anything other than temporary demos - skipFinalSnapshot: true - storageEncrypted: false - allocatedStorage: 10 - #! storageType: gp2 #! One of "standard" (magnetic), "gp2" (general purpose SSD), or "io1" (provisioned IOPS SSD). The default is "io1" if iops is specified, "gp2" if not. - #!optionGroupName: abc #! needs reference/api resouce - #!parameterGroupName: abc #! needs reference/api resouce - vpcSecurityGroupIdSelector: - matchControllerRef: true - dbSubnetGroupNameSelector: - matchControllerRef: true - providerConfigRef: - name: #@ data.values.providers.aws.configRef - patches: - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.forProvider.passwordSecretRef.name - transforms: - - type: string - string: - type: Format - fmt: '%s-postgresql-admin' - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.version - toFieldPath: spec.forProvider.engineVersion - transforms: - - type: map - map: - "11": "11.17" - "12": "12.12" - "13": "13.8" - "14": "14.5" - "15": "14.5" - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.location - toFieldPath: spec.forProvider.region - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.adminUsername - toFieldPath: spec.forProvider.username - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.database - toFieldPath: spec.forProvider.dbName - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.storageClass - toFieldPath: spec.forProvider.storageType - #! not allowed when using 11, 12, not sure about others - #! - type: FromCompositeFieldPath - #! fromFieldPath: spec.parameters.charset - #! toFieldPath: spec.forProvider.characterSetName - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.collation - toFieldPath: spec.forProvider.collation - - connectionDetails: - - name: type - value: postgresql - - name: provider - value: aws - - name: port - value: "5432" - - name: database - fromFieldPath: spec.forProvider.dbName - type: FromFieldPath - - name: username - fromFieldPath: spec.forProvider.username - type: FromFieldPath - - name: host - fromFieldPath: status.atProvider.address - type: FromFieldPath - - - - name: securitygroup - base: - apiVersion: ec2.aws.upbound.io/v1beta1 - kind: SecurityGroup - patches: - - type: FromCompositeFieldPath - fromFieldPath: metadata.name - toFieldPath: spec.forProvider.name - - type: FromCompositeFieldPath - fromFieldPath: metadata.name - toFieldPath: spec.forProvider.description - transforms: - - string: - fmt: 'Traffic to RDS instance %s' - type: Format - type: string - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.vpcId - toFieldPath: spec.forProvider.vpcId - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.location - toFieldPath: spec.forProvider.region - - - name: securitygrouprule - base: - apiVersion: ec2.aws.upbound.io/v1beta1 - kind: SecurityGroupRule - spec: - forProvider: - fromPort: 5432 - toPort: 5432 - protocol: tcp - securityGroupIdSelector: - matchControllerRef: true - type: ingress - patches: - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.cidrBlocks - toFieldPath: spec.forProvider.cidrBlocks - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.location - toFieldPath: spec.forProvider.region - - type: FromCompositeFieldPath - fromFieldPath: metadata.labels.port - toFieldPath: spec.forProvider.fromPort - transforms: - - type: convert - convert: - toType: int - - type: FromCompositeFieldPath - fromFieldPath: metadata.labels.port - toFieldPath: spec.forProvider.toPort - transforms: - - type: convert - convert: - toType: int - - - name: subnetgroup - base: - apiVersion: rds.aws.upbound.io/v1beta1 - kind: SubnetGroup - spec: - forProvider: - subnetIdSelector: - matchControllerRef: true - tags: - Name: subnet group for RDS via crossplane - patches: - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: metadata.name - transforms: - - type: string - string: - type: Format - fmt: 'rds-sg-%s' - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.location - toFieldPath: spec.forProvider.region - - - name: subneta - base: - apiVersion: ec2.aws.upbound.io/v1beta1 - kind: Subnet - patches: - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.vpcId - toFieldPath: spec.forProvider.vpcId - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.location - toFieldPath: spec.forProvider.region - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.subnetACidrBlock - toFieldPath: spec.forProvider.cidrBlock - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: metadata.name - transforms: - - type: string - string: - type: Format - fmt: 'subnet-a-%s' - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.location - toFieldPath: spec.forProvider.availabilityZone - transforms: - - type: string - string: - type: Format - fmt: '%sa' - - name: subnetb - base: - apiVersion: ec2.aws.upbound.io/v1beta1 - kind: Subnet - patches: - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.vpcId - toFieldPath: spec.forProvider.vpcId - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.location - toFieldPath: spec.forProvider.region - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.subnetBCidrBlock - toFieldPath: spec.forProvider.cidrBlock - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: metadata.name - transforms: - - type: string - string: - type: Format - fmt: 'subnet-b-%s' - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.location - toFieldPath: spec.forProvider.availabilityZone - transforms: - - type: string - string: - type: Format - fmt: '%sb' - - name: routetable - base: - apiVersion: ec2.aws.upbound.io/v1beta1 - kind: RouteTable - metadata: - name: table - spec: - forProvider: - tags: - Name: RDS-via-Crossplane - patches: - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.vpcId - toFieldPath: spec.forProvider.vpcId - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.location - toFieldPath: spec.forProvider.region - - name: routetableassoA - base: - apiVersion: ec2.aws.upbound.io/v1beta1 - kind: RouteTableAssociation - metadata: - name: rta - spec: - forProvider: - routeTableIdSelector: - matchControllerRef: true - patches: - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.location - toFieldPath: spec.forProvider.region - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: metadata.name - transforms: - - type: string - string: - type: Format - fmt: 'subnet-a-%s' - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.forProvider.subnetIdRef.name - transforms: - - type: string - string: - type: Format - fmt: 'subnet-a-%s' - - name: routetableassoB - base: - apiVersion: ec2.aws.upbound.io/v1beta1 - kind: RouteTableAssociation - metadata: - name: rta - spec: - forProvider: - routeTableIdSelector: - matchControllerRef: true - patches: - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.location - toFieldPath: spec.forProvider.region - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: metadata.name - transforms: - - type: string - string: - type: Format - fmt: 'subnet-b-%s' - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.forProvider.subnetIdRef.name - transforms: - - type: string - string: - type: Format - fmt: 'subnet-b-%s' - - name: route - base: - apiVersion: ec2.aws.upbound.io/v1beta1 - kind: Route - metadata: - name: route - spec: - forProvider: - destinationCidrBlock: 0.0.0.0/0 - routeTableIdSelector: - matchControllerRef: true - patches: - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.location - toFieldPath: spec.forProvider.region - - type: FromCompositeFieldPath - fromFieldPath: spec.parameters.gatewayId - toFieldPath: spec.forProvider.gatewayId - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: metadata.name - transforms: - - type: string - string: - type: Format - fmt: 'route-%s' \ No newline at end of file diff --git a/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml index 4c294f8..a2e51c5 100644 --- a/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/definition.ytt.yml @@ -27,16 +27,20 @@ spec: properties: spec: type: object + required: + - parameters properties: parameters: type: object + required: + - location properties: location: type: string version: type: string default: "13" - enum: ["11", "12", "13"] + enum: ["11", "12", "13", "14", "15"] adminUsername: type: string default: postgres @@ -49,27 +53,6 @@ spec: collation: type: string default: en_US.utf8 - gatewayId: - type: string - description: (only public AWS) - subnetACidrBlock: - type: string - description: (only AWS) - subnetBCidrBlock: - type: string - description: (only AWS) - cidrBlocks: - type: array - description: (only AWS) cidr blocks for AWS security group - items: - type: string - pattern: '^(1?\d{1,2}|2[01234]\d|25[012345])(\.(1?\d{1,2}|2[01234]\d|25[012345])){3}\/(\d|[12]\d|3[012])$' - dbSubnetGroupName: - type: string - description: (only AWS) name for the AWS subnet group - vpcId: - type: string - description: (only AWS) id of the AWS vpc charset: type: string default: utf8 @@ -80,10 +63,47 @@ spec: type: string endIpAddress: type: string - required: - - location - required: - - parameters + aws: + type: object + description: properties for AWS only + required: + - cidrBlocks + - dbSubnetGroupName + - vpcId + properties: + cidrBlocks: + type: array + description: cidr blocks for AWS security group + default: + - "0.0.0.0/0" + items: + type: string + pattern: '^(1?\d{1,2}|2[01234]\d|25[012345])(\.(1?\d{1,2}|2[01234]\d|25[012345])){3}\/(\d|[12]\d|3[012])$' + dbSubnetGroupName: + type: string + description: name for the AWS subnet group + vpcId: + type: string + description: id of the existing vpc + public: + type: object + description: set these if you use the AWS public composition + required: + - gatewayId + - subnetACidrBlock + - subnetBCidrBlock + properties: + gatewayId: + type: string + description: id of the existing gateway + subnetACidrBlock: + type: string + description: cidr block for the subnet a (creates two subnets, a and b) + default: "10.100.255.0/25" + subnetBCidrBlock: + type: string + description: cidr block for the subnet b (creates two subnets, a and b) + default: "10.100.255.128/25" status: type: object properties: From b60dcd776566142d5df1cac633b9af3937bae57d Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Thu, 1 Dec 2022 10:40:33 +0100 Subject: [PATCH 16/41] fix crossplane aws provider name --- docs/crossplane/providers/aws.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/crossplane/providers/aws.md b/docs/crossplane/providers/aws.md index b1958eb..b29ab11 100644 --- a/docs/crossplane/providers/aws.md +++ b/docs/crossplane/providers/aws.md @@ -23,7 +23,7 @@ and store the desired release into the `PROVIDER_AWS_RELEASE` variable. === "Upbound CLI" Do make sure you have installed the `up` CLI as described [here](../index.md) and execute ```sh - up controlplane provider install xpkg.upbound.io/upbound/provider-aws:${PROVIDER_AWS_RELEASE} --name provider-aws + up controlplane provider install xpkg.upbound.io/upbound/provider-aws:${PROVIDER_AWS_RELEASE} --name upbound-provider-aws ``` === "YAML manifest" @@ -32,7 +32,7 @@ and store the desired release into the `PROVIDER_AWS_RELEASE` variable. apiVersion: pkg.crossplane.io/v1 kind: Provider metadata: - name: provider-aws + name: upbound-provider-aws spec: package: xpkg.upbound.io/upbound/provider-aws:${PROVIDER_AWS_RELEASE} EOF @@ -87,7 +87,7 @@ cat > ${ROLE_TRUST_POLICY} < Date: Tue, 3 Jan 2023 21:23:31 +0100 Subject: [PATCH 17/41] forgot to update helm and azure compositions to the upbound tf provider --- .../multicloud/crossplane/psql/ytt/azure-composition.ytt.yml | 4 ++-- .../multicloud/crossplane/psql/ytt/helm-composition.ytt.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml index 8dd73b8..1850f34 100644 --- a/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml @@ -21,7 +21,7 @@ spec: spec: forProvider: manifest: - apiVersion: tf.crossplane.io/v1alpha1 + apiVersion: tf.upbound.io/v1beta1 kind: ProviderConfig spec: credentials: [] @@ -48,7 +48,7 @@ spec: toFieldPath: spec.forProvider.manifest.metadata.name - name: password base: - apiVersion: tf.crossplane.io/v1alpha1 + apiVersion: tf.upbound.io/v1beta1 kind: Workspace spec: forProvider: diff --git a/packages/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml index 6b1f234..0052572 100644 --- a/packages/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml @@ -22,7 +22,7 @@ spec: spec: forProvider: manifest: - apiVersion: tf.crossplane.io/v1alpha1 + apiVersion: tf.upbound.io/v1beta1 kind: ProviderConfig spec: credentials: [] @@ -52,7 +52,7 @@ spec: - name: password base: - apiVersion: tf.crossplane.io/v1alpha1 + apiVersion: tf.upbound.io/v1beta1 kind: Workspace spec: forProvider: From d539c5605ca008ebd030df37ed7116a2d79ed876 Mon Sep 17 00:00:00 2001 From: Joost van der Griendt Date: Wed, 4 Jan 2023 17:32:26 +0100 Subject: [PATCH 18/41] add labels to secret and use shared library --- .../psql/ytt/aws-composition-private.ytt.yml | 4 +- .../psql/ytt/aws-composition-public.ytt.yml | 4 +- .../psql/ytt/aws-composition.lib.yml | 80 ------------- .../psql/ytt/azure-composition.ytt.yml | 80 +------------ .../psql/ytt/helm-composition.ytt.yml | 87 +------------- .../crossplane/psql/ytt/shared.lib.yml | 111 ++++++++++++++++++ 6 files changed, 129 insertions(+), 237 deletions(-) create mode 100644 packages/multicloud/crossplane/psql/ytt/shared.lib.yml diff --git a/packages/multicloud/crossplane/psql/ytt/aws-composition-private.ytt.yml b/packages/multicloud/crossplane/psql/ytt/aws-composition-private.ytt.yml index 7dbdfe0..b43fcc3 100644 --- a/packages/multicloud/crossplane/psql/ytt/aws-composition-private.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/aws-composition-private.ytt.yml @@ -1,6 +1,7 @@ #@ load("@ytt:data", "data") --- -#@ load("aws-composition.lib.yml", "tfProviderConfig", "tfWorkspace", "rdsInstance", "securityGroup", "securityGroupRule") +#@ load("aws-composition.lib.yml", "rdsInstance", "securityGroup", "securityGroupRule") +#@ load("shared.lib.yml", "labelsForSecret", "tfProviderConfig", "tfWorkspace") apiVersion: apiextensions.crossplane.io/v1 kind: Composition @@ -17,6 +18,7 @@ spec: apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version kind: #@ data.values.xrd.names.kind resources: + - #@ labelsForSecret("aws") - #@ tfProviderConfig(data.values.crossplane.namespace) - #@ tfWorkspace(data.values.crossplane.namespace) - #@ rdsInstance(data.values.crossplane.namespace, data.values.providers.aws.configRef, False) diff --git a/packages/multicloud/crossplane/psql/ytt/aws-composition-public.ytt.yml b/packages/multicloud/crossplane/psql/ytt/aws-composition-public.ytt.yml index 93fc3c3..68d16ed 100644 --- a/packages/multicloud/crossplane/psql/ytt/aws-composition-public.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/aws-composition-public.ytt.yml @@ -1,6 +1,7 @@ #@ load("@ytt:data", "data") --- -#@ load("aws-composition.lib.yml", "tfProviderConfig", "tfWorkspace", "rdsInstance", "securityGroup", "securityGroupRule", "subnet", "routeTableAssociation", "routeTable", "route", "subnetGroup") +#@ load("aws-composition.lib.yml", "rdsInstance", "securityGroup", "securityGroupRule", "subnet", "routeTableAssociation", "routeTable", "route", "subnetGroup") +#@ load("shared.lib.yml", "labelsForSecret", "tfProviderConfig", "tfWorkspace") #@ subnetASuffix = '-a' #@ subnetAFormat = 'subnet-a-%s' @@ -27,6 +28,7 @@ spec: apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version kind: #@ data.values.xrd.names.kind resources: + - #@ labelsForSecret("aws") - #@ tfProviderConfig(data.values.crossplane.namespace) - #@ tfWorkspace(data.values.crossplane.namespace) - #@ rdsInstance(data.values.crossplane.namespace, data.values.providers.aws.configRef, True) diff --git a/packages/multicloud/crossplane/psql/ytt/aws-composition.lib.yml b/packages/multicloud/crossplane/psql/ytt/aws-composition.lib.yml index db3370d..bba188b 100644 --- a/packages/multicloud/crossplane/psql/ytt/aws-composition.lib.yml +++ b/packages/multicloud/crossplane/psql/ytt/aws-composition.lib.yml @@ -2,86 +2,6 @@ #@ end -#@ def tfProviderConfig(crossplaneNamespace): -name: tf-providerconfig -base: - apiVersion: kubernetes.crossplane.io/v1alpha1 - kind: Object - spec: - forProvider: - manifest: - apiVersion: tf.upbound.io/v1beta1 - kind: ProviderConfig - spec: - credentials: [] - #@yaml/text-templated-strings - configuration: | - terraform { - required_providers { - random = { - source = "hashicorp/random" - version = "3.4.3" - } - } - - backend "kubernetes" { - secret_suffix = "providerconfig-default" - namespace = "(@= crossplaneNamespace @)" - in_cluster_config = true - } - } - provider "random" {} -patches: -- type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.forProvider.manifest.metadata.name -#@ end - - -#@ def tfWorkspace(crossplaneNamespace): -name: password -base: - apiVersion: tf.upbound.io/v1beta1 - kind: Workspace - spec: - forProvider: - module: | - resource "random_password" "password" { - length = 64 - special = false - } - - output "password" { - value = random_password.password.result - sensitive = true - } - source: Inline - writeConnectionSecretToRef: - namespace: #@ crossplaneNamespace -patches: -- type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.writeConnectionSecretToRef.name - transforms: - - type: string - string: - type: Format - fmt: '%s-postgresql-admin' -- type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.providerConfigRef.name -- type: ToCompositeFieldPath - fromFieldPath: metadata.labels[crossplane.io/claim-name] - toFieldPath: spec.writeConnectionSecretToRef.name -- type: ToCompositeFieldPath - fromFieldPath: metadata.labels[crossplane.io/claim-namespace] - toFieldPath: spec.writeConnectionSecretToRef.namespace -- type: ToCompositeFieldPath - fromFieldPath: metadata.labels[crossplane.io/claim-name] - toFieldPath: status.binding.name -connectionDetails: -- fromConnectionSecretKey: password -#@ end #@ def rdsInstance(crossplaneNamespace, providerConfigRef, publiclyAccessible): name: rdsinstance diff --git a/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml index 1850f34..27693c8 100644 --- a/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/azure-composition.ytt.yml @@ -1,5 +1,8 @@ #@ load("@ytt:data", "data") --- + +#@ load("shared.lib.yml", "labelsForSecret", "tfProviderConfig", "tfWorkspace") + apiVersion: apiextensions.crossplane.io/v1 kind: Composition metadata: @@ -14,80 +17,9 @@ spec: apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version kind: #@ data.values.xrd.names.kind resources: - - name: tf-providerconfig - base: - apiVersion: kubernetes.crossplane.io/v1alpha1 - kind: Object - spec: - forProvider: - manifest: - apiVersion: tf.upbound.io/v1beta1 - kind: ProviderConfig - spec: - credentials: [] - #@yaml/text-templated-strings - configuration: | - terraform { - required_providers { - random = { - source = "hashicorp/random" - version = "3.4.3" - } - } - - backend "kubernetes" { - secret_suffix = "providerconfig-default" - namespace = "(@= data.values.crossplane.namespace @)" - in_cluster_config = true - } - } - provider "random" {} - patches: - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.forProvider.manifest.metadata.name - - name: password - base: - apiVersion: tf.upbound.io/v1beta1 - kind: Workspace - spec: - forProvider: - module: | - resource "random_password" "password" { - length = 64 - special = false - } - - output "password" { - value = random_password.password.result - sensitive = true - } - source: Inline - writeConnectionSecretToRef: - namespace: #@ data.values.crossplane.namespace - patches: - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.writeConnectionSecretToRef.name - transforms: - - type: string - string: - type: Format - fmt: '%s-postgresql-admin' - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.providerConfigRef.name - - type: ToCompositeFieldPath - fromFieldPath: metadata.labels[crossplane.io/claim-name] - toFieldPath: spec.writeConnectionSecretToRef.name - - type: ToCompositeFieldPath - fromFieldPath: metadata.labels[crossplane.io/claim-namespace] - toFieldPath: spec.writeConnectionSecretToRef.namespace - - type: ToCompositeFieldPath - fromFieldPath: metadata.labels[crossplane.io/claim-name] - toFieldPath: status.binding.name - connectionDetails: - - fromConnectionSecretKey: password + - #@ labelsForSecret("azure") + - #@ tfProviderConfig(data.values.crossplane.namespace) + - #@ tfWorkspace(data.values.crossplane.namespace) - name: resourcegroup base: apiVersion: azure.upbound.io/v1beta1 diff --git a/packages/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml index 0052572..470bc47 100644 --- a/packages/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml @@ -1,5 +1,8 @@ #@ load("@ytt:data", "data") --- + +#@ load("shared.lib.yml", "labelsForSecret", "tfProviderConfig", "tfWorkspace") + apiVersion: apiextensions.crossplane.io/v1 kind: Composition metadata: @@ -14,87 +17,9 @@ spec: apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version kind: #@ data.values.xrd.names.kind resources: - - - name: tf-providerconfig - base: - apiVersion: kubernetes.crossplane.io/v1alpha1 - kind: Object - spec: - forProvider: - manifest: - apiVersion: tf.upbound.io/v1beta1 - kind: ProviderConfig - spec: - credentials: [] - #@yaml/text-templated-strings - configuration: | - terraform { - required_providers { - random = { - source = "hashicorp/random" - version = "3.4.3" - } - } - - backend "kubernetes" { - secret_suffix = "providerconfig-default" - namespace = "(@= data.values.crossplane.namespace @)" - in_cluster_config = true - } - } - - provider "random" {} - - patches: - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.forProvider.manifest.metadata.name - - - name: password - base: - apiVersion: tf.upbound.io/v1beta1 - kind: Workspace - spec: - forProvider: - module: | - resource "random_password" "password" { - length = 64 - special = false - } - - output "password" { - value = random_password.password.result - sensitive = true - } - source: Inline - writeConnectionSecretToRef: - namespace: #@ data.values.crossplane.namespace - - patches: - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.writeConnectionSecretToRef.name - transforms: - - type: string - string: - type: Format - fmt: '%s-postgresql-admin' - - type: FromCompositeFieldPath - fromFieldPath: metadata.uid - toFieldPath: spec.providerConfigRef.name - - type: ToCompositeFieldPath - fromFieldPath: metadata.labels[crossplane.io/claim-name] - toFieldPath: spec.writeConnectionSecretToRef.name - - type: ToCompositeFieldPath - fromFieldPath: metadata.labels[crossplane.io/claim-namespace] - toFieldPath: spec.writeConnectionSecretToRef.namespace - - type: ToCompositeFieldPath - fromFieldPath: metadata.labels[crossplane.io/claim-name] - toFieldPath: status.binding.name - - connectionDetails: - - fromConnectionSecretKey: password - + - #@ labelsForSecret("kubernetes") + - #@ tfProviderConfig(data.values.crossplane.namespace) + - #@ tfWorkspace(data.values.crossplane.namespace) - name: release base: apiVersion: helm.crossplane.io/v1beta1 diff --git a/packages/multicloud/crossplane/psql/ytt/shared.lib.yml b/packages/multicloud/crossplane/psql/ytt/shared.lib.yml new file mode 100644 index 0000000..39ec00a --- /dev/null +++ b/packages/multicloud/crossplane/psql/ytt/shared.lib.yml @@ -0,0 +1,111 @@ + +#@ def labelsForSecret(infra): +name: connectionSecret +base: + apiVersion: kubernetes.crossplane.io/v1alpha1 + kind: Object + spec: + forProvider: + manifest: + apiVersion: v1 + kind: Secret + spec: {} + metadata: + labels: + services.apps.tanzu.vmware.com/class: multicloud-psql + services.apps.tanzu.vmware.com/infra: #@ infra +patches: +- type: FromCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-name] + toFieldPath: spec.forProvider.manifest.metadata.name +- type: FromCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-namespace] + toFieldPath: spec.forProvider.manifest.metadata.namespace +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.version + toFieldPath: spec.forProvider.manifest.metadata.labels[services.apps.tanzu.vmware.com/version] +- type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.manifest.metadata.labels[services.apps.tanzu.vmware.com/location] +#@ end + +#@ def tfProviderConfig(crossplaneNamespace): +name: tf-providerconfig +base: + apiVersion: kubernetes.crossplane.io/v1alpha1 + kind: Object + spec: + forProvider: + manifest: + apiVersion: tf.upbound.io/v1beta1 + kind: ProviderConfig + spec: + credentials: [] + #@yaml/text-templated-strings + configuration: | + terraform { + required_providers { + random = { + source = "hashicorp/random" + version = "3.4.3" + } + } + + backend "kubernetes" { + secret_suffix = "providerconfig-default" + namespace = "(@= crossplaneNamespace @)" + in_cluster_config = true + } + } + provider "random" {} +patches: +- type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.forProvider.manifest.metadata.name +#@ end + + +#@ def tfWorkspace(crossplaneNamespace): +name: password +base: + apiVersion: tf.upbound.io/v1beta1 + kind: Workspace + spec: + forProvider: + module: | + resource "random_password" "password" { + length = 64 + special = false + } + + output "password" { + value = random_password.password.result + sensitive = true + } + source: Inline + writeConnectionSecretToRef: + namespace: #@ crossplaneNamespace +patches: +- type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.writeConnectionSecretToRef.name + transforms: + - type: string + string: + type: Format + fmt: '%s-postgresql-admin' +- type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.providerConfigRef.name +- type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-name] + toFieldPath: spec.writeConnectionSecretToRef.name +- type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-namespace] + toFieldPath: spec.writeConnectionSecretToRef.namespace +- type: ToCompositeFieldPath + fromFieldPath: metadata.labels[crossplane.io/claim-name] + toFieldPath: status.binding.name +connectionDetails: +- fromConnectionSecretKey: password +#@ end From 80a598f416b87c2f3cd3c8db94932a32b5d66604 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Tue, 3 Jan 2023 11:57:06 +0100 Subject: [PATCH 19/41] fix missing variable definition --- docs/usecases/aws/prerequisites/eks.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/usecases/aws/prerequisites/eks.md b/docs/usecases/aws/prerequisites/eks.md index e6f70d4..0e79405 100644 --- a/docs/usecases/aws/prerequisites/eks.md +++ b/docs/usecases/aws/prerequisites/eks.md @@ -68,8 +68,9 @@ EBS volumes. ```sh # define variables for IAM ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) -OIDC_ID=$(aws eks describe-cluster --name ${CLUSTER_NAME} --output text --query "cluster.identity.oidc.issuer" | cut -d/ -f3-) -EBS_ROLE=AmazonEKS_EBS_CSI_Driver-${CLUSTER_NAME} +OIDC_URL=$(aws eks describe-cluster --name ${CLUSTER_NAME} --output text --query "cluster.identity.oidc.issuer") +OIDC_ID=$(cut -d/ -f3- <<<$OIDC_URL) +EBS_ROLE="AmazonEKS_EBS_CSI_Driver-${CLUSTER_NAME}" ROLE_TRUST_POLICY=$(mktemp) ROLE_PERMISSION_POLICY=$(mktemp) From dbb3c67cf84461d6c041133ffe3ff11fbf62e097 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Thu, 1 Dec 2022 16:36:12 +0100 Subject: [PATCH 20/41] build and release carvel packages and repository --- .github/workflows/publish-carvel-repo.yml | 32 +++ .github/workflows/publish-packages.yml | 101 ++++++- .github/workflows/reusable-bump-version.yml | 7 +- .github/workflows/reusable-carvel-publish.yml | 92 +++++-- .../reusable-carvel-repo-publish.yml | 78 ++++++ .github/workflows/reusable-carvel-test.yml | 111 ++++++-- .../workflows/reusable-crossplane-publish.yml | 11 +- .github/workflows/reusable-list-packages.yml | 7 +- .gitignore | 4 + Makefile | 207 ++++++++------- config/README.md | 3 + config/carvel/README.md | 0 .../package-build/package-build.ytt.yml | 39 +++ config/carvel/package-install/rbac.ytt.yml | 52 ++++ config/carvel/package-install/schema.ytt.yml | 9 + .../package-resources/package-resources.yml | 6 + .../pkgrepo-build/pkgrepo-build.ytt.yml | 10 + packagerepo.yaml | 12 - packages/.gitignore | 4 + packages/README.md | 114 ++++++++ .../aws/carvel/elasticache/build-values.yml | 7 + .../carvel/elasticache/config/00-schema.yml | 64 +++++ .../config/01-replication-group.ytt.yml | 221 ++++++++++++++++ .../elasticache/config/99-kapp-config.yml | 43 +++ .../carvel/elasticache/package-metadata.yml | 17 ++ .../azure/carvel/psql/config/00-schema.yml | 140 ++++++++++ .../psql/config/01-flexible-server.ytt.yml | 246 ++++++++++++++++++ .../carvel/psql/config/99-kapp-config.yml | 53 ++++ .../azure/carvel/psql/package-metadata.yml | 9 +- repository/.imgpkg/images.yml | 28 -- .../packages/amazon/elasticache/metadata.yml | 17 -- .../packages/amazon/elasticache/package.yml | 122 --------- repository/packages/amazon/rds/meta.yaml | 15 -- repository/packages/amazon/rds/package.yaml | 57 ---- repository/packages/azure/psql/package.yml | 246 ------------------ repository/packages/google/cloudsql/meta.yaml | 15 -- .../packages/google/cloudsql/package.yaml | 101 ------- scripts/carvel-azure-install-aso.sh | 33 +++ scripts/carvel-e2e-azure-psql.sh | 78 ++++++ .../carvel-e2e-azure-psql/app-overlay.ytt.yml | 28 ++ scripts/carvel-e2e-azure-psql/cleanup.sh | 26 ++ scripts/carvel-e2e-azure-psql/rbac.ytt.yml | 53 ++++ scripts/carvel-e2e-azure-psql/test.sh | 42 +++ ...odb.sh => crossplane-e2e-azure-mongodb.sh} | 0 .../claim-instance.sh | 0 .../cleanup.sh | 0 .../install-package.sh | 0 .../test.sh | 0 48 files changed, 1796 insertions(+), 764 deletions(-) create mode 100644 .github/workflows/publish-carvel-repo.yml create mode 100644 .github/workflows/reusable-carvel-repo-publish.yml create mode 100644 config/README.md create mode 100644 config/carvel/README.md create mode 100644 config/carvel/package-build/package-build.ytt.yml create mode 100644 config/carvel/package-install/rbac.ytt.yml create mode 100644 config/carvel/package-install/schema.ytt.yml create mode 100644 config/carvel/package-resources/package-resources.yml create mode 100644 config/carvel/pkgrepo-build/pkgrepo-build.ytt.yml delete mode 100644 packagerepo.yaml create mode 100644 packages/.gitignore create mode 100644 packages/README.md create mode 100644 packages/aws/carvel/elasticache/build-values.yml create mode 100644 packages/aws/carvel/elasticache/config/00-schema.yml create mode 100644 packages/aws/carvel/elasticache/config/01-replication-group.ytt.yml create mode 100644 packages/aws/carvel/elasticache/config/99-kapp-config.yml create mode 100644 packages/aws/carvel/elasticache/package-metadata.yml create mode 100644 packages/azure/carvel/psql/config/00-schema.yml create mode 100644 packages/azure/carvel/psql/config/01-flexible-server.ytt.yml create mode 100644 packages/azure/carvel/psql/config/99-kapp-config.yml rename repository/packages/azure/psql/metadata.yml => packages/azure/carvel/psql/package-metadata.yml (58%) delete mode 100644 repository/.imgpkg/images.yml delete mode 100644 repository/packages/amazon/elasticache/metadata.yml delete mode 100644 repository/packages/amazon/elasticache/package.yml delete mode 100644 repository/packages/amazon/rds/meta.yaml delete mode 100644 repository/packages/amazon/rds/package.yaml delete mode 100644 repository/packages/azure/psql/package.yml delete mode 100644 repository/packages/google/cloudsql/meta.yaml delete mode 100644 repository/packages/google/cloudsql/package.yaml create mode 100755 scripts/carvel-azure-install-aso.sh create mode 100755 scripts/carvel-e2e-azure-psql.sh create mode 100644 scripts/carvel-e2e-azure-psql/app-overlay.ytt.yml create mode 100755 scripts/carvel-e2e-azure-psql/cleanup.sh create mode 100644 scripts/carvel-e2e-azure-psql/rbac.ytt.yml create mode 100755 scripts/carvel-e2e-azure-psql/test.sh rename scripts/{crossplane-e2e-mongodb.sh => crossplane-e2e-azure-mongodb.sh} (100%) rename scripts/{crossplane-e2e-mongodb => crossplane-e2e-azure-mongodb}/claim-instance.sh (100%) rename scripts/{crossplane-e2e-mongodb => crossplane-e2e-azure-mongodb}/cleanup.sh (100%) rename scripts/{crossplane-e2e-mongodb => crossplane-e2e-azure-mongodb}/install-package.sh (100%) rename scripts/{crossplane-e2e-mongodb => crossplane-e2e-azure-mongodb}/test.sh (100%) diff --git a/.github/workflows/publish-carvel-repo.yml b/.github/workflows/publish-carvel-repo.yml new file mode 100644 index 0000000..2994c15 --- /dev/null +++ b/.github/workflows/publish-carvel-repo.yml @@ -0,0 +1,32 @@ +name: Publish Carvel repository + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - main + paths: + - 'repository/packages/*/*.yml' + + pull_request: + types: [opened, ready_for_review, reopened, synchronize] + branches: + - main + - gha-carvel + paths: + - 'repository/packages/*/*.yml' + +jobs: + + bump-version: + uses: ./.github/workflows/reusable-bump-version.yml + + carvel-repo-publish: + needs: + - bump-version + uses: ./.github/workflows/reusable-carvel-repo-publish.yml + with: + version: ${{ needs.bump-version.outputs.version }} diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index 2e4674f..6629c38 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -8,6 +8,7 @@ on: push: branches: - main + - gha-carvel paths: - 'packages/*/*/*/**' @@ -22,6 +23,8 @@ jobs: list-packages: uses: ./.github/workflows/reusable-list-packages.yml + with: + basedir: packages bump-version: needs: @@ -41,6 +44,8 @@ jobs: package_name: ${{ matrix.name }} package_provider: ${{ matrix.provider }} package_version: ${{ needs.bump-version.outputs.version }} + packages_basedir: ${{ needs.list-packages.outputs.basedir }} + package_path: ${{ matrix.path }} run-test: ${{ github.event_name == 'pull_request' }} carvel-publish: @@ -55,4 +60,98 @@ jobs: package_name: ${{ matrix.name }} package_provider: ${{ matrix.provider }} package_version: ${{ needs.bump-version.outputs.version }} - run-test: ${{ github.event_name == 'pull_request' }} + packages_basedir: ${{ needs.list-packages.outputs.basedir }} + package_path: ${{ matrix.path }} + + carvel-prepare-repo: + needs: + - bump-version + - list-packages + - carvel-publish + runs-on: ubuntu-latest + + outputs: + repo-branch-name: ${{ steps.git.outputs.REPO_BRANCH_NAME }} + + env: + PACKAGE_VERSION: ${{ needs.bump-version.outputs.version }} + REPO_BRANCH_NAME: carvel-repo-${{ needs.bump-version.outputs.version }} + + steps: + + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: '0' + + - name: Prepare git client + id: git + run: | + git config --global user.email "${{ github.event.repository.name }}@${{ github.repository_owner }}.github.io" + git config --global user.name "GitHub Actions Workflow" + + sleep 5 && git fetch --all + + echo "CURRENT_BRANCH=$(git branch --show-current)" >> $GITHUB_ENV + echo "REPO_BRANCH_NAME=$REPO_BRANCH_NAME" >> $GITHUB_OUTPUT + + - name: Create branch and pull request + run: | + # create new branch + git checkout -b ${REPO_BRANCH_NAME} + + # include the changes from all the packages' branches + # and remove them at the end of the script + BRANCHES= + for PACKAGE_DIR in $(jq -r '.include[]|.path' <<<$PACKAGES_LIST); do + PACKAGE_METADATA_NAME=$(yq e '.metadata.name' ${PACKAGES_BASEDIR}/${PACKAGE_DIR}/package-metadata.yml) + PACKAGE_BRANCH="${PACKAGE_METADATA_NAME}/${PACKAGE_VERSION}" + BRANCHES="${BRANCHES} ${PACKAGE_BRANCH}" + git rebase origin/${PACKAGE_BRANCH} + done + trap "git push --delete origin ${BRANCHES}" EXIT + + # prepare a proper commit message + COMMIT_MESSAGE="$(git log --pretty=format:'%s' ${CURRENT_BRANCH}..HEAD)" + + # discard the existing commits from packages' branches but do keep the changed files + git reset --soft ${CURRENT_BRANCH} + + # create a new commit on the repo branch and push it to origin + git add . && git commit -m "${COMMIT_MESSAGE}" && git push -u origin ${REPO_BRANCH_NAME} + + # create the pull request against CURRENT_BRANCH + gh pr create --base ${CURRENT_BRANCH} --title "Carvel repository release at commit ${{ github.sha }}" --body "${COMMIT_MESSAGE}" + env: + PACKAGES_BASEDIR: ${{ needs.list-packages.outputs.basedir }} + PACKAGES_LIST: ${{ needs.list-packages.outputs.carvel }} + GH_TOKEN: ${{ github.token }} + + carvel-repo-publish: + if: needs.list-packages.outputs.carvel_publish == 'true' + uses: ./.github/workflows/reusable-carvel-repo-publish.yml + needs: + - bump-version + - carvel-prepare-repo + with: + git_ref: ${{ needs.carvel-prepare-repo.outputs.repo-branch-name }} + repo_version: ${{ needs.bump-version.outputs.version }} + release: true + + carvel-test: + if: needs.list-packages.outputs.carvel_publish == 'true' + uses: ./.github/workflows/reusable-carvel-test.yml + needs: + - list-packages + - bump-version + - carvel-repo-publish + strategy: + matrix: ${{ fromJson(needs.list-packages.outputs.carvel) }} + with: + repo_version: ${{ needs.bump-version.outputs.version }} + package_name: ${{ matrix.name }} + package_provider: ${{ matrix.provider }} + package_version: ${{ needs.bump-version.outputs.version }} + packages_basedir: ${{ needs.list-packages.outputs.basedir }} + package_path: ${{ matrix.path }} + secrets: inherit diff --git a/.github/workflows/reusable-bump-version.yml b/.github/workflows/reusable-bump-version.yml index fd848bd..d9b0edb 100644 --- a/.github/workflows/reusable-bump-version.yml +++ b/.github/workflows/reusable-bump-version.yml @@ -4,6 +4,11 @@ on: workflow_call: + inputs: + default_bump: + type: string + default: patch + outputs: version: value: ${{ jobs.bump-version.outputs.version }} @@ -32,4 +37,4 @@ jobs: INITIAL_VERSION: 0.1.0 RELEASE_BRANCHES: main PRERELEASE: true - DEFAULT_BUMP: patch + DEFAULT_BUMP: ${{ inputs.default_bump }} diff --git a/.github/workflows/reusable-carvel-publish.yml b/.github/workflows/reusable-carvel-publish.yml index 445fce5..9ae4814 100644 --- a/.github/workflows/reusable-carvel-publish.yml +++ b/.github/workflows/reusable-carvel-publish.yml @@ -5,6 +5,13 @@ on: workflow_call: inputs: + packages_basedir: + type: string + description: Packages base directory path + default: packages + package_path: + type: string + required: true package_version: type: string required: true @@ -14,21 +21,38 @@ on: package_provider: type: string required: true - run-test: - type: boolean - default: false + repo_dir: + type: string + description: Directory path to hold the contents of the Carvel repository + default: repository kubernetes-version: type: string default: v1.24.6 kind-version: type: string default: v0.16.0 + kctrl-version: + type: string + default: v0.43.2 outputs: package_repository: value: ${{ jobs.publish.outputs.package_repository }} package_registry: value: ${{ jobs.publish.outputs.package_registry }} + package_metadata_name: + value: ${{ jobs.publish.outputs.package_metadata_name }} + +env: + PACKAGE_VERSION: ${{ inputs.package_version }} + PACKAGE_REGISTRY: ghcr.io + PACKAGE_REPOSITORY: ${{ github.repository }}/${{ inputs.package_provider }}/carvel/${{ inputs.package_name }} + PACKAGE_PROVIDER: ${{ inputs.package_provider }} + PACKAGE_NAME: ${{ inputs.package_name }} + PACKAGES_BASEDIR: ${{ inputs.packages_basedir }} + PACKAGE_DIR: ${{ inputs.packages_basedir }}/${{ inputs.package_path }} + + CARVEL_REPO_DIR: ${{ inputs.repo_dir }} jobs: @@ -38,28 +62,48 @@ jobs: outputs: package_repository: ${{ env.PACKAGE_REPOSITORY }} package_registry: ${{ env.PACKAGE_REGISTRY }} - - env: - PACKAGE_VERSION: ${{ inputs.package_version }} - PACKAGE_REGISTRY: ghcr.io - PACKAGE_REPOSITORY: ${{ github.repository }}/${{ inputs.package_provider }}/carvel/${{ inputs.package_name }} - PACKAGE_PROVIDER: ${{ inputs.package_provider }} - PACKAGE_NAME: ${{ inputs.package_name }} + package_metadata_name: ${{ steps.info.outputs.package_metadata_name }} steps: - - name: Dump info + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: '0' + + - name: Install Carvel tools + uses: vmware-tanzu/carvel-setup-action@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + kctrl: ${{ inputs.kctrl-version }} + + - name: Kctrl Release + run: make kctrl-release + env: + IMGPKG_REGISTRY_HOSTNAME: ${{ env.PACKAGE_REGISTRY }} + IMGPKG_REGISTRY_USERNAME: ${{ github.actor }} + IMGPKG_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + + - name: Get package info + id: info run: | - env | sort - - test: - if: inputs.run-test - uses: ./.github/workflows/reusable-carvel-test.yml - needs: - - publish - with: - package_version: ${{ inputs.package_version }} - package_name: ${{ inputs.package_name }} - package_provider: ${{ inputs.package_provider }} - package_registry: ${{ needs.publish.outputs.package_registry }} - package_repository: ${{ needs.publish.outputs.package_repository }} + package_metadata_name=$(yq e '.metadata.name' ${PACKAGE_DIR}/package-metadata.yml) + echo "package_metadata_name=${package_metadata_name}" >> $GITHUB_OUTPUT + + # The files for the repository have been generated into ${CARVEL_REPO_DIR}/packages + # They will be added to a dedicated branch + - name: Push artifacts + run: | + CURRENT_BRANCH=$(git branch --show-current) + ARTIFACTS_BRANCH_NAME=${PACKAGE_METADATA_NAME}/${PACKAGE_VERSION} + ARTIFACTS_DIR=${CARVEL_REPO_DIR}/packages/${PACKAGE_METADATA_NAME} + + git config --global user.email "${{ github.event.repository.name }}@${{ github.repository_owner }}.github.io" + git config --global user.name "GitHub Actions Workflow" + + git checkout -b ${ARTIFACTS_BRANCH_NAME} + git add ${ARTIFACTS_DIR} + git commit -m "${ARTIFACTS_BRANCH_NAME}" + git push -u origin ${ARTIFACTS_BRANCH_NAME} + env: + PACKAGE_METADATA_NAME: ${{ steps.info.outputs.package_metadata_name }} diff --git a/.github/workflows/reusable-carvel-repo-publish.yml b/.github/workflows/reusable-carvel-repo-publish.yml new file mode 100644 index 0000000..108d8ad --- /dev/null +++ b/.github/workflows/reusable-carvel-repo-publish.yml @@ -0,0 +1,78 @@ +name: Reusable workflow for publishing Carvel packages repository + +on: + + workflow_call: + + inputs: + repo_dir: + type: string + default: repository + release: + type: boolean + default: false + repo_name: + type: string + default: carvel-reference-packages + repo_version: + type: string + git_ref: + type: string + default: ${{ github.ref }} + description: Git ref to checkout (defaults to github.ref) + kctrl-version: + type: string + default: v0.43.2 + + outputs: + registry: + value: ${{ jobs.publish.outputs.registry }} + repository: + value: ${{ jobs.publish.outputs.repository }} + +env: + CARVEL_REPO_DIR: ${{ inputs.repo_dir }} + CARVEL_REPO_NAME: ${{ inputs.repo_name }} + CARVEL_REPO_REGISTRY: ghcr.io + CARVEL_REPO_REPOSITORY: ${{ github.repository }}/${{ inputs.repo_name }} + CARVEL_REPO_VERSION: ${{ inputs.repo_version }} + +jobs: + + publish: + runs-on: ubuntu-latest + + outputs: + registry: ${{ env.CARVEL_REPO_REGISTRY }} + repository: ${{ env.CARVEL_REPO_REPOSITORY }} + + steps: + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ inputs.git_ref }} + + - name: Install Carvel tools + uses: vmware-tanzu/carvel-setup-action@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + kctrl: ${{ inputs.kctrl-version }} + + - name: Kctrl Release + id: kctrl + run: | + make kctrl-repo-release + env: + IMGPKG_REGISTRY_HOSTNAME: ${{ env.CARVEL_REPO_REGISTRY }} + IMGPKG_REGISTRY_USERNAME: ${{ github.actor }} + IMGPKG_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + + - name: Release + if: inputs.release + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ env.CARVEL_REPO_VERSION }} + fail_on_unmatched_files: true + files: | + repository/package-repository.yml diff --git a/.github/workflows/reusable-carvel-test.yml b/.github/workflows/reusable-carvel-test.yml index 173ff04..b0e7070 100644 --- a/.github/workflows/reusable-carvel-test.yml +++ b/.github/workflows/reusable-carvel-test.yml @@ -5,6 +5,9 @@ on: workflow_call: inputs: + repo_version: + type: string + required: true package_version: type: string required: true @@ -14,10 +17,11 @@ on: package_provider: type: string required: true - package_registry: + packages_basedir: type: string - required: true - package_repository: + description: Packages base directory path + default: packages + package_path: type: string required: true kubernetes-version: @@ -26,21 +30,31 @@ on: kind-version: type: string default: v0.16.0 + kapp-controller-version: + type: string + default: v0.43.2 + secretgen-controller-version: + type: string + default: v0.12.0 + + secrets: + AZURE_CONFIG: + required: false jobs: test: runs-on: ubuntu-latest + env: - CROSSPLANE_NAMESPACE: upbound-system - CONFIG_VERSION: ${{ inputs.package_version }} - CONFIG_IMAGE: ${{ inputs.package_registry }}/${{ inputs.package_repository }} + PACKAGE_VERSION: ${{ inputs.package_version }} steps: - - name: Dump info - run: | - env | sort + - name: Install Carvel tools + uses: vmware-tanzu/carvel-setup-action@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Create k8s Kind Cluster uses: helm/kind-action@v1.4.0 @@ -57,21 +71,82 @@ jobs: kubectl cluster-info kubectl get storageclass standard - - name: Create Azure Config Secret + - name: Install kapp-controller + run: | + MANIFEST="https://github.com/vmware-tanzu/carvel-kapp-controller/releases/download/${{ inputs.kapp-controller-version }}/release.yml" + if [[ "${{ inputs.kapp-controller-version }}" == "latest" ]]; then + MANIFEST="https://github.com/vmware-tanzu/carvel-kapp-controller/releases/latest/download/release.yml" + fi + + kubectl apply -f ${MANIFEST} + time kubectl wait --for=condition=Available=True -n kapp-controller deploy kapp-controller + time kubectl wait --for=condition=Available=True apiservices.apiregistration.k8s.io v1alpha1.data.packaging.carvel.dev + + # determine the kapp-controller global namespace + echo KAPP_GLOBAL_NAMESPACE=$(kubectl -n kapp-controller get deployment kapp-controller -o json | jq -r '.spec.template.spec.containers[]|select(.name=="kapp-controller").args[]|select(.|startswith("-packaging-global-namespace"))|split("=")[1]') >> $GITHUB_ENV + + - name: Install secretgen-controller + run: | + MANIFEST="https://github.com/vmware-tanzu/carvel-secretgen-controller/releases/download/${{ inputs.secretgen-controller-version }}/release.yml" + if [[ "${{ inputs.secretgen-controller-version }}" == "latest" ]]; then + MANIFEST="https://github.com/vmware-tanzu/carvel-secretgen-controller/releases/latest/download/release.yml" + fi + + kubectl apply -f ${MANIFEST} + time kubectl wait --for=condition=Available=True -n secretgen-controller deploy secretgen-controller + time kubectl wait --for=condition=Available=True apiservices.apiregistration.k8s.io v1alpha1.secretgen.carvel.dev + + - name: Install Carvel repository + run: | + REPO_MANIFEST="https://github.com/${{ github.repository }}/releases/download/${{ inputs.repo_version }}/package-repository.yml" + REPO_MANIFEST_FILE=$(mktemp) + curl -sSfL -o ${REPO_MANIFEST_FILE} ${REPO_MANIFEST} + REPO_NAME=$(yq '.metadata.name' ${REPO_MANIFEST_FILE}) + kubectl apply -n ${KAPP_GLOBAL_NAMESPACE} -f ${REPO_MANIFEST_FILE} + time kubectl wait --for=condition=ReconcileSucceeded=True packagerepositories.packaging.carvel.dev -n ${KAPP_GLOBAL_NAMESPACE} ${REPO_NAME} + + - name: List available packages + run: | + kubectl get packages.data.packaging.carvel.dev + + - name: Mask secret + if: inputs.package_provider == 'azure' + uses: matteo-magni/secret-mask-action@main + with: + secret: ${{ secrets.AZURE_CONFIG }} + + - name: Set Azure secret if: inputs.package_provider == 'azure' + env: + AZURE_CONFIG: ${{ secrets.AZURE_CONFIG }} run: | - kubectl create namespace ${CROSSPLANE_NAMESPACE} - kubectl create secret generic azure-secret -n ${CROSSPLANE_NAMESPACE} --from-literal=creds='${{ secrets.AZURE_CONFIG }}' + # transforms the keys in the input JSON secret from camelCase to UPPER_SNAKE_CASE + # with the AZURE_ prefix + # in order not to divert from the official ASO docs and the client JSON file generated by az CLI + for k in $(jq -r 'keys[]' <<<"$AZURE_CONFIG"); do + K=$(echo azure_$k | sed -r 's/([a-z0-9])([A-Z])/\1_\L\2/g' | tr '[:lower:]' '[:upper:]') + echo $K=$(jq -r '.'$k <<<"$AZURE_CONFIG") >> $GITHUB_ENV + done + + - name: Checkout + uses: actions/checkout@v3 - - name: Test Carvel Package + - name: Test Carvel package run: | - [ -x ${SCRIPT} ] && ${SCRIPT} || true + export PACKAGE_METADATA_NAME=$(yq e '.metadata.name' ${PACKAGE_DIR}/package-metadata.yml) + echo PACKAGE_METADATA_NAME=${PACKAGE_METADATA_NAME} >> $GITHUB_ENV + if [ -x ${SCRIPT} ]; then + ${SCRIPT} + fi env: - SCRIPT: ./scripts/carvel-e2e-${{ inputs.package_name }}.sh + SCRIPT: ./scripts/carvel-e2e-${{ inputs.package_provider }}-${{ inputs.package_name }}.sh + PACKAGE_DIR: ${{ inputs.packages_basedir }}/${{ inputs.package_path }} - - name: Cleanup Carvel Package + - name: Cleanup Carvel package if: always() run: | - [ -x ${SCRIPT} ] && ${SCRIPT} || true + if [ -x ${SCRIPT} ]; then + ${SCRIPT} + fi env: - SCRIPT: ./scripts/carvel-e2e-${{ inputs.package_name }}/cleanup.sh + SCRIPT: ./scripts/carvel-e2e-${{ inputs.package_provider }}-${{ inputs.package_name }}/cleanup.sh diff --git a/.github/workflows/reusable-crossplane-publish.yml b/.github/workflows/reusable-crossplane-publish.yml index a29ea0e..405180a 100644 --- a/.github/workflows/reusable-crossplane-publish.yml +++ b/.github/workflows/reusable-crossplane-publish.yml @@ -5,6 +5,13 @@ on: workflow_call: inputs: + packages_basedir: + type: string + description: Packages base directory path + default: packages + package_path: + type: string + required: true package_version: type: string required: true @@ -45,7 +52,9 @@ jobs: PACKAGE_REPOSITORY: ${{ github.repository }}/${{ inputs.package_provider }}/crossplane/${{ inputs.package_name }} PACKAGE_PROVIDER: ${{ inputs.package_provider }} PACKAGE_NAME: ${{ inputs.package_name }} - + PACKAGES_BASEDIR: ${{ inputs.packages_basedir }} + PACKAGE_DIR: ${{ inputs.packages_basedir }}/${{ inputs.package_path }} + steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/reusable-list-packages.yml b/.github/workflows/reusable-list-packages.yml index e3dcd49..cb2841c 100644 --- a/.github/workflows/reusable-list-packages.yml +++ b/.github/workflows/reusable-list-packages.yml @@ -15,6 +15,8 @@ on: default: HEAD~1 outputs: + basedir: + value: ${{ inputs.basedir }} all: value: ${{ jobs.list-packages.outputs.all }} crossplane: @@ -56,7 +58,8 @@ jobs: # that have been either Added, Copied, Modified or Renamed (ACMR). # This list is fed to `jq` that transforms it into an actual json array, filters by the entries that start with "BASEDIR/" # and strip the prefix. - # Each entry is then tokenised by /, morphed into an object with "provider" key set as the first token, "packaging" as the second one and "name" as the third one. + # Each entry is then tokenised by /, morphed into an object with "provider" key set as the first token, "packaging" as the second one, "name" as the third one + # and the "path" as them all back together. # The resulting array is then purged of the duplicates and then set as value for the "include" key in a new object, # in order to meet https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#expanding-or-adding-matrix-configurations. # The result is appended to GITHUB_OUTPUT as well as written to the workflow's output for debugging purposes (thus the `tee -a`). @@ -67,7 +70,7 @@ jobs: [inputs] | map(capture("^"+$BASEDIR+"/(?

.*)$").p | split("/") - | { "provider": .[0], "packaging": .[1], "name": .[2] } + | { "path": .[0:3]|join("/"), "provider": .[0], "packaging": .[1], "name": .[2] } ) | map(select(.provider!=null and .packaging!=null and .name!=null)) | unique diff --git a/.gitignore b/.gitignore index fa06fd1..8b63b0a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ # Except this file !.gitignore + /site/ +**/.src/ +**/.package/ +pkgrepo-build.yml diff --git a/Makefile b/Makefile index 9d9e056..349e13f 100644 --- a/Makefile +++ b/Makefile @@ -1,107 +1,116 @@ -LOCAL_VERSION = $(shell git describe --tags --always) -PACKAGE_VERSION ?= "0.0.0-${LOCAL_VERSION}" -PACKAGE_REPOSITORY ?= "vmware-tanzu-labs/trp-azure-psql" -PACKAGE_REGISTRY ?= "ghcr.io" -CLOUD_NAME = azure - -PACKAGES_DIR = ./packages -PACKAGE_DIR = ${PACKAGES_DIR}/${PACKAGE_NAME} -PACKAGE_BUILD_BASE_DIR = ./package-build -PACKAGE_BUILD_DIR = ${PACKAGE_BUILD_BASE_DIR}/${PACKAGE_NAME} -PACKAGE_METADATA_DIR = ${PACKAGE_DIR}/package-metadata -PACKAGES_REPO_STAGING_DIR = ./packages-repo -TEMPLATES_DIR = ${PACKAGE_DIR}/package-templates - -PACKAGE_TEST_DATA_DIR_NAME = test-data -REPOSITORY_DIR = ../tap-reference-packages-repository -REPOSITORY_CLOUD_DIR = ${REPOSITORY_DIR}/repository/packages/${CLOUD_NAME} - -SHELL := $(shell which bash) - -KDEV_SA_NAME=development -KDEV_SA_NAMESPACE=default - -DEV_PACKAGE_NAME ?= ${PACKAGE_NAME} -DEV_NAMESPACE ?= default - -# validate-input: -# @# echo "PACKAGE_NAME is ${PACKAGE_NAME}" -# @# echo "PACKAGES_DIR is ${PACKAGES_DIR}" -# @# echo ""${PACKAGES_DIR}/${PACKAGE_NAME}" is directory:" $(shell [ -d "${PACKAGES_DIR}/${PACKAGE_NAME}" ] && echo "true" || echo "false") -# @[ -z "${PACKAGE_NAME}" ] && (echo "PACKAGE_NAME is empty" && exit 1) -# @[ -d "${PACKAGES_DIR}/${PACKAGE_NAME}" ] || $(shell echo "${PACKAGES_DIR}/${PACKAGE_NAME} directory does not exist" && exit 2) - -clean: - rm -f ${PACKAGE_DIR}/package-kdev.yml || true - rm -f ${PACKAGE_DIR}/package-build.yml || true - rm -f ${PACKAGE_DIR}/package-resources.yml || true - rm -rf ${PACKAGE_BUILD_DIR} || true - rm -rf ${PACKAGE_DIR}/carvel-artifacts/ || true - rm -rf ${PACKAGE_DIR}/bundle-* || true - rm -rf ${PACKAGES_REPO_STAGING_DIR} || true - mkdir -p ${PACKAGES_REPO_STAGING_DIR} - -dev: clean - kubectl create namespace ${DEV_PACKAGE_NAME} || true - ytt -f ${PACKAGE_DIR}/config -f ${PACKAGE_DIR}/test-data | kbld -f - | kapp deploy -a ${DEV_PACKAGE_NAME} -n ${DEV_NAMESPACE} --debug -y -f - - -dev-cleanup: - kapp delete -a ${DEV_PACKAGE_NAME} -n ${DEV_NAMESPACE} -y - -kdev-prepare: clean - yq e '... comments="" | .spec.template.spec.template[] |= select(.ytt != null).ytt.paths += "test-data"' ${PACKAGE_DIR}/package.yml > ${PACKAGE_DIR}/package-kdev.yml - cd ${PACKAGE_DIR} && ytt -f package-kdev.yml -f package-metadata.yml -f package-install.yml > package-resources.yml - -kdev: kdev-prepare - cd ${PACKAGE_DIR} && kctrl dev -f package-resources.yml -l - -kdev-cleanup: kdev-prepare - cd ${PACKAGE_DIR} && kctrl dev -f package-resources.yml -l --delete - -imgpkg-push-prepare: - mkdir -p ${PACKAGE_BUILD_DIR}/.imgpkg - cp -a ${PACKAGE_DIR}/config ${PACKAGE_BUILD_DIR} - kbld -f ${PACKAGE_BUILD_DIR}/config/ --imgpkg-lock-output ${PACKAGE_BUILD_DIR}/.imgpkg/images.yml - -imgpkg-push: imgpkg-push-prepare - imgpkg push -b ${PACKAGE_REGISTRY}/${PACKAGE_REPOSITORY}:${PACKAGE_VERSION} -f ${PACKAGE_BUILD_DIR}/ - -release-prepare: - cd ${PACKAGE_DIR} && ytt -f package.yml -f package-metadata.yml -f package-install.yml > package-resources.yml - cd ${PACKAGE_DIR} && ytt -f package-build.ytt.yml -f package-build-schema.yml \ - -v package_fqdn=$(shell yq e .metadata.name ${PACKAGE_DIR}/package-metadata.yml) \ - -v repository=${PACKAGE_REPOSITORY} -v registry=${PACKAGE_REGISTRY} > package-build.yml - mkdir -p ${PACKAGE_DIR}/.imgpkg - kbld -f ${PACKAGE_DIR}/config/ --imgpkg-lock-output ${PACKAGE_DIR}/.imgpkg/images.yml - -# Does not use the version, and thus cannot be used until kctrl fixes that -kctrl-release: release-prepare - kctrl package release --chdir "${PACKAGE_DIR}" --version="${PACKAGE_VERSION}" --repo-output="${PWD}/${PACKAGES_REPO_STAGING_DIR}" -y - -create-kdev-sa: - kubectl create serviceaccount ${KDEV_SA_NAME} -n ${KDEV_SA_NAMESPACE} || true - kubectl delete clusterrolebinding kdev-cluster-admin || true - kubectl create clusterrolebinding kdev-cluster-admin --clusterrole=cluster-admin --serviceaccount=${KDEV_SA_NAMESPACE}:${KDEV_SA_NAME} - -repo-prepare: - mkdir -p ${REPOSITORY_CLOUD_DIR} - -repo-package-copy: repo-prepare - cp -a ${PACKAGES_REPO_STAGING_DIR}/packages/$(shell yq e .metadata.name ${PACKAGE_DIR}/package-metadata.yml)/ ${REPOSITORY_CLOUD_DIR}/${PACKAGE_NAME} +# LOCAL_VERSION = $(shell git describe --tags --always) +# PACKAGE_VERSION ?= "0.0.0-${LOCAL_VERSION}" +# PACKAGE_REPOSITORY ?= "vmware-tanzu-labs/trp-azure-psql" +# PACKAGE_REGISTRY ?= "ghcr.io" +# CLOUD_NAME = azure +# +# PACKAGES_DIR ?= ./packages +# PACKAGE_DIR ?= ${PACKAGES_DIR}/${PACKAGE_NAME} +# RELEASE_DIR = ${PACKAGE_DIR}/.release +# +# PACKAGE_BUILD_BASE_DIR = ./package-build +# PACKAGE_BUILD_DIR = ${PACKAGE_BUILD_BASE_DIR}/${PACKAGE_NAME} +# +# PACKAGE_TEST_DATA_DIR_NAME = test-data +# REPOSITORY_DIR = ../tap-reference-packages-repository +# REPOSITORY_CLOUD_DIR = ${REPOSITORY_DIR}/repository/packages/${CLOUD_NAME} +# +# SHELL := $(shell which bash) +# +# KDEV_SA_NAME=development +# KDEV_SA_NAMESPACE=default +# +# DEV_PACKAGE_NAME ?= ${PACKAGE_NAME} +# DEV_NAMESPACE ?= default + + + +# clean: +# rm -f ${PACKAGE_DIR}/package-kdev.yml || true +# rm -f ${PACKAGE_DIR}/package-build.yml || true +# rm -f ${PACKAGE_DIR}/package-resources.yml || true +# rm -rf ${PACKAGE_BUILD_DIR} || true +# rm -rf ${PACKAGE_DIR}/carvel-artifacts/ || true +# rm -rf ${PACKAGE_DIR}/bundle-* || true +# rm -rf ${PACKAGES_REPO_STAGING_DIR} || true +# mkdir -p ${PACKAGES_REPO_STAGING_DIR} +# +# dev: clean +# kubectl create namespace ${DEV_PACKAGE_NAME} || true +# ytt -f ${PACKAGE_DIR}/config -f ${PACKAGE_DIR}/test-data | kbld -f - | kapp deploy -a ${DEV_PACKAGE_NAME} -n ${DEV_NAMESPACE} --debug -y -f - +# +# dev-cleanup: +# kapp delete -a ${DEV_PACKAGE_NAME} -n ${DEV_NAMESPACE} -y +# +# kdev-prepare: clean +# yq e '... comments="" | .spec.template.spec.template[] |= select(.ytt != null).ytt.paths += "test-data"' ${PACKAGE_DIR}/package.yml > ${PACKAGE_DIR}/package-kdev.yml +# cd ${PACKAGE_DIR} && ytt -f package-kdev.yml -f package-metadata.yml -f package-install.yml > package-resources.yml +# +# kdev: kdev-prepare +# cd ${PACKAGE_DIR} && kctrl dev -f package-resources.yml -l +# +# kdev-cleanup: kdev-prepare +# cd ${PACKAGE_DIR} && kctrl dev -f package-resources.yml -l --delete +# +# create-kdev-sa: +# kubectl create serviceaccount ${KDEV_SA_NAME} -n ${KDEV_SA_NAMESPACE} || true +# kubectl delete clusterrolebinding kdev-cluster-admin || true +# kubectl create clusterrolebinding kdev-cluster-admin --clusterrole=cluster-admin --serviceaccount=${KDEV_SA_NAMESPACE}:${KDEV_SA_NAME} +# +# repo-prepare: +# mkdir -p ${REPOSITORY_CLOUD_DIR} +# +# repo-package-copy: repo-prepare +# cp -a ${PACKAGES_REPO_STAGING_DIR}/packages/$(shell yq e .metadata.name ${PACKAGE_DIR}/package-metadata.yml)/ ${REPOSITORY_CLOUD_DIR}/${PACKAGE_NAME} + + + +# imgpkg-push-prepare: +# mkdir -p ${PACKAGE_BUILD_DIR}/.imgpkg +# cp -a ${PACKAGE_DIR}/config ${PACKAGE_BUILD_DIR} +# kbld -f ${PACKAGE_BUILD_DIR}/config/ --imgpkg-lock-output ${PACKAGE_BUILD_DIR}/.imgpkg/images.yml +# +# imgpkg-push: imgpkg-push-prepare +# imgpkg push -b ${PACKAGE_REGISTRY}/${PACKAGE_REPOSITORY}:${PACKAGE_VERSION} -f ${PACKAGE_BUILD_DIR}/ + +PACKAGES_BASEDIR ?= packages +PACKAGE_DIR ?= ${PACKAGES_BASEDIR}/${PACKAGE_PROVIDER}/${PACKAGE_PACKAGING}/${PACKAGE_NAME} + +CARVEL_REPO_DIR ?= repository + +#TODO purge older versions according to a TBD retention policy + +kctrl-release-prepare: + ytt --data-values-file ${PACKAGE_DIR}/package-metadata.yml -f config/carvel/package-resources -f ${PACKAGE_DIR}/package-metadata.yml -v version=${PACKAGE_VERSION} > ${PACKAGE_DIR}/package-resources.yml + ytt --data-values-file ${PACKAGE_DIR}/package-metadata.yml -f config/carvel/package-build -v registry=${PACKAGE_REGISTRY} -v repository=${PACKAGE_REPOSITORY} > ${PACKAGE_DIR}/package-build.yml + +kctrl-release: kctrl-release-prepare + kctrl package release --chdir "${PACKAGE_DIR}" --version="${PACKAGE_VERSION}" --tag="${PACKAGE_VERSION}" --repo-output="${PWD}/${CARVEL_REPO_DIR}" -y + +kctrl-repo-release-prepare: + mkdir -p ${CARVEL_REPO_DIR} + ytt -f config/carvel/pkgrepo-build -v name=${CARVEL_REPO_NAME} -v registry=${CARVEL_REPO_REGISTRY} -v repository=${CARVEL_REPO_REPOSITORY} > ${CARVEL_REPO_DIR}/pkgrepo-build.yml + +kctrl-repo-release: kctrl-repo-release-prepare + kctrl package repository release --chdir ${CARVEL_REPO_DIR} -v ${CARVEL_REPO_VERSION} -y + +# yq -i '.spec.fetch.imgpkgBundle.image=(.spec.fetch.imgpkgBundle.image|split("@")|.[0])+":"+.metadata.annotations."kctrl.carvel.dev/repository-version"' ${REPO_BASEDIR}/package-repository.yml crossplane-ytt: - mkdir -p ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/src/ - ytt -f ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/ytt > ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/src/crossplane.yml + rm -rf ${PACKAGE_DIR}/.src || true + mkdir -p ${PACKAGE_DIR}/.src + + ytt -f ${PACKAGE_DIR}/ytt > ${PACKAGE_DIR}/.src/crossplane.yml crossplane-build: crossplane-ytt - mkdir -p ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package - rm ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package/*.xpkg || true + rm -rf ${PACKAGE_DIR}/.package || true + mkdir -p ${PACKAGE_DIR}/.package + up xpkg build \ - --package-root ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/src \ - --examples-root ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/claim-examples \ - --output ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package/crossplane-${PACKAGE_NAME}.xpkg + --package-root ${PACKAGE_DIR}/.src \ + --examples-root ${PACKAGE_DIR}/claim-examples \ + --output ${PACKAGE_DIR}/.package/crossplane.xpkg crossplane-push: crossplane-build - up xpkg push --package ${PACKAGES_DIR}/${PACKAGE_PROVIDER}/crossplane/${PACKAGE_NAME}/package/crossplane-${PACKAGE_NAME}.xpkg \ + up xpkg push --package ${PACKAGE_DIR}/.package/crossplane.xpkg \ ${PACKAGE_REGISTRY}/${PACKAGE_REPOSITORY}:${PACKAGE_VERSION} \ - --domain ${PACKAGE_REGISTRY} \ No newline at end of file diff --git a/config/README.md b/config/README.md new file mode 100644 index 0000000..f6e1b3b --- /dev/null +++ b/config/README.md @@ -0,0 +1,3 @@ +# Configuration + +This directory contains files used for configuring either [the Carvel suite](./carvel) or [Crossplane](./crossplane) to build packages. diff --git a/config/carvel/README.md b/config/carvel/README.md new file mode 100644 index 0000000..e69de29 diff --git a/config/carvel/package-build/package-build.ytt.yml b/config/carvel/package-build/package-build.ytt.yml new file mode 100644 index 0000000..7251f79 --- /dev/null +++ b/config/carvel/package-build/package-build.ytt.yml @@ -0,0 +1,39 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:assert", "assert") + +#@ if data.values.registry == "": +#@ assert.fail('data value "registry" must not be empty') +#@ end +#@ if data.values.repository == "": +#@ assert.fail('data value "repository" must not be empty') +#@ end +#@ if data.values.metadata.name == "": +#@ assert.fail('.metadata.name must not be empty') +#@ end + +--- +apiVersion: kctrl.carvel.dev/v1alpha1 +kind: PackageBuild +metadata: + creationTimestamp: null + name: #@ data.values.metadata.name +spec: + release: + - resource: {} + template: + spec: + app: + spec: + deploy: + - kapp: {} + template: + - ytt: + paths: + - config + - kbld: {} + export: + - imgpkgBundle: + image: #@ data.values.registry + "/" + data.values.repository + useKbldImagesLock: true + includePaths: + - config diff --git a/config/carvel/package-install/rbac.ytt.yml b/config/carvel/package-install/rbac.ytt.yml new file mode 100644 index 0000000..e7a6dc6 --- /dev/null +++ b/config/carvel/package-install/rbac.ytt.yml @@ -0,0 +1,52 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:overlay", "overlay") + +#@ serviceAccountName = data.values.refName + "-install" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: #@ serviceAccountName +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: #@ serviceAccountName +rules: +- apiGroups: ["dbforpostgresql.azure.com"] + resources: ["flexibleservers","flexibleserversdatabases","flexibleserversfirewallrules"] + verbs: ["*"] +- apiGroups: ["resources.azure.com"] + resources: ["resourcegroups"] + verbs: ["*"] +- apiGroups: ["secretgen.carvel.dev", "secretgen.k14s.io"] + resources: ["secrettemplates","passwords"] + verbs: ["*"] +- apiGroups: [""] + resources: ["serviceaccounts","configmaps"] + verbs: ["*"] +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles","rolebindings"] + verbs: ["*"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: #@ serviceAccountName + "-binding" +subjects: +- kind: ServiceAccount + name: #@ serviceAccountName +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: #@ serviceAccountName + + +#@overlay/match by=overlay.all, expects="1+" +--- +metadata: + #@overlay/match missing_ok=True + namespace: #@ data.values.namespace diff --git a/config/carvel/package-install/schema.ytt.yml b/config/carvel/package-install/schema.ytt.yml new file mode 100644 index 0000000..936d365 --- /dev/null +++ b/config/carvel/package-install/schema.ytt.yml @@ -0,0 +1,9 @@ +#@data/values-schema +--- +#@schema/desc "The package full name as from .spec.refName" +#@schema/validation min_len=1 +refName: "" + +#@schema/desc "The namespace where the package must be installed" +#@schema/validation min_len=1 +namespace: services diff --git a/config/carvel/package-resources/package-resources.yml b/config/carvel/package-resources/package-resources.yml new file mode 100644 index 0000000..e56224c --- /dev/null +++ b/config/carvel/package-resources/package-resources.yml @@ -0,0 +1,6 @@ +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +--- +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: Package diff --git a/config/carvel/pkgrepo-build/pkgrepo-build.ytt.yml b/config/carvel/pkgrepo-build/pkgrepo-build.ytt.yml new file mode 100644 index 0000000..c8c14c4 --- /dev/null +++ b/config/carvel/pkgrepo-build/pkgrepo-build.ytt.yml @@ -0,0 +1,10 @@ +#@ load("@ytt:data", "data") +--- +apiVersion: kctrl.carvel.dev/v1alpha1 +kind: PackageRepositoryBuild +metadata: + name: #@ data.values.name +spec: + export: + imgpkgBundle: + image: #@ data.values.registry + "/" + data.values.repository diff --git a/packagerepo.yaml b/packagerepo.yaml deleted file mode 100644 index 1d0cd8c..0000000 --- a/packagerepo.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -apiVersion: packaging.carvel.dev/v1alpha1 -kind: PackageRepository -metadata: - name: tap-reference-service-packages - namespace: tanzu-package-repo-global #! NOTE this may be different if kapp was not installed through cluster-essentials - #! the kapp-controller global namespace can be obtained as follows - #! kubectl -n kapp-controller get deployment kapp-controller -o json | jq -r '.spec.template.spec.containers[]|select(.name=="kapp-controller").args[]|select(.|startswith("-packaging-global-namespace"))|split("=")[1]' -spec: - fetch: - imgpkgBundle: - image: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages:0.0.4-alpha.0 diff --git a/packages/.gitignore b/packages/.gitignore new file mode 100644 index 0000000..af5beeb --- /dev/null +++ b/packages/.gitignore @@ -0,0 +1,4 @@ +carvel-artifacts/ +bundle-*/ +package-resources.yml +package-build.yml diff --git a/packages/README.md b/packages/README.md new file mode 100644 index 0000000..7ef20f4 --- /dev/null +++ b/packages/README.md @@ -0,0 +1,114 @@ +# Packages structure + +This is the parent folder holding the configurations the reference packages will be built from. +In order for the CI/CD workflow to work, they must adhere to the following structure: + +```text +packages +├── +│ └── +│ └── +│ ├── ... +│ ├── ... +┆ ┆ +``` + +- `PROVIDER`: the provider the package is built for (i.e. `aws`, `azure`, `gcp`, `multicloud`). +- `PACKAGING`: the packaging system used (currently it must be either `carvel` or `crossplane`). +- `NAME`: the package name. + +All folder names must be lowercase. + +An actual example, featuring AWS Elasticache package built with Carvel suite and a multicloud PostgreSQL package with Crossplane, looks like the following: + +```text +packages +├── aws +│ └── carvel +│ └── elasticache +│ ├── build-values.yml +│ ├── config +│ │ ├── 00-schema.yml +│ │ ├── 01-replication-group.ytt.yml +│ │ └── 99-kapp-config.yml +│ └── package-metadata.yml +┆ +└── multicloud + └── crossplane + └── postgresql + ├── README.md + ├── claim + │ └── helm.yml + ├── claim-examples + │ └── helm-psql-12.yaml + └── ytt + ├── crossplane.ytt.yml + ├── definition.ytt.yml + ├── helm-composition.ytt.yml + └── schema.ytt.yml +``` + +## Packaging + +The contents of the `//` directory depends on the specific packaging system. + +### Carvel + +The `kctrl` utility from [Carvel suite](https://carvel.dev) is being used to author packages, which makes use of other tools in the suite as well (i.e. `ytt`). + +The metadata files required to create the package are stored as [ytt templates](../config/carvel/) +to easily define a standard to create multiple packages. + +**N.B. Each package MUST have its own `PackageMetadata` manifest stored in the +`package-metadata.yml` file, located in the package's main directory.** + +This file will be included in the actual package as well as used for holding information +that is required to fill some fields in other template files, in a DRY fashion. + +*DRY: Don't Repeat Yourself + +The `config` directory inside the package home holds the ytt files that define the resources that will be created by the package installation +as well as the schema definition for the input values. + +**N.B. The name of such a folder MUST be `config`, as it is hardcoded in the mentioned templates.** + +If some sort of validation of the input data has been implemented in the ytt files, you also need +a sample values file to feed `kctrl` with at building time, otherwise it won't succeed. +Those sample data are just used at this stage and won't be included in the package itself. +The sample values file name, if required, MUST be stored into the `PACKAGE_BUILD_VALUES` variable. + +**N.B. By default, the CI workflow assumes that such is called `build-values.yml`, therefore packages +undergoing continuous integration MUST use this name.** + +The command used to build the package and publish it to the container registry is `make kctrl-release`, which uses a few environment variables +for configuration. +The following snippet defines a quick way of building all the `carvel` packages in the `packages` folder. +Additionally, it uploads the packages to the ghcr.io service in the `org/repository` repository (do adjust it to your own account), +in a hierarchy that reflects the filesystem. + +```sh +PACKAGE_REGISTRY="ghcr.io" +PACKAGE_BUILD_VALUES="build-values.yml" +for PACKAGE_DIR in $(find packages -type d -mindepth 3 -maxdepth 3); do + if [[ $(cut -d/ -f3 <<<${PACKAGE_DIR}) == "carvel" ]]; then + PACKAGE_REPOSITORY="org/repository/${PACKAGE_DIR#packages/}" + make kctrl-release + fi +done +``` + +The prerequisites to that are that you must be authenticated to ghcr.io and you must have the correct permissions +to write packages to `org`. +The authentication credentials can be provided either via [Docker login][docker-login] or via [`imgpkg` environment variables][imgpkg-auth-env], for example: + +```sh +export IMGPKG_REGISTRY_HOSTNAME="ghcr.io" +export IMGPKG_REGISTRY_USERNAME="my-username" +export IMGPKG_REGISTRY_PASSWORD="my-personal-access-token" +``` + +[imgpkg-auth-env]: https://carvel.dev/imgpkg/docs/v0.34.0/auth/#via-environment-variables +[docker-login]: https://docs.docker.com/engine/reference/commandline/login/ + +### Crossplane + diff --git a/packages/aws/carvel/elasticache/build-values.yml b/packages/aws/carvel/elasticache/build-values.yml new file mode 100644 index 0000000..64dc4d5 --- /dev/null +++ b/packages/aws/carvel/elasticache/build-values.yml @@ -0,0 +1,7 @@ +#@data/values +--- +name: elasticache +namespace: default +cacheSubnetGroupName: default +vpcSecurityGroupIDs: + - vpc-0123456789 diff --git a/packages/aws/carvel/elasticache/config/00-schema.yml b/packages/aws/carvel/elasticache/config/00-schema.yml new file mode 100644 index 0000000..6b37539 --- /dev/null +++ b/packages/aws/carvel/elasticache/config/00-schema.yml @@ -0,0 +1,64 @@ +#@data/values-schema +--- +#@schema/title "Name" +#@schema/desc "Name of the Elasticache instance." +name: "" + +#@schema/title "Namespace" +#@schema/desc "Namespace to deploy the kubernetes resources to." +namespace: "" + +#@schema/title "CreateNamespace" +#@schema/desc "Whether to create the namespace for the resources or not." +createNamespace: False + +#@schema/title "CacheSubnetGroupName" +#@schema/desc "Name of the cache subnet group to create the replication group in. It must be pre-created if the createCacheSubnetGroup parameter is set to false." +cacheSubnetGroupName: "" + +#@schema/title "CreateCacheSubnetGroup" +#@schema/desc "Whether to create the CacheSubnetGroup or not." +createCacheSubnetGroup: false + +#@schema/title "SubnetIDs" +#@schema/desc "The list of subnets to create the CacheSubnetGroup for. Mandatory if the createCacheSubnetGroup parameter is set to true." +subnetIDs: + - "" + +#@schema/title "CacheNodeType" +#@schema/desc "Type of nodes used for the replication group." +cacheNodeType: "cache.t2.micro" + +#@schema/title "VpcSecurityGroupIDs" +#@schema/desc "The list of security groups to associate to the replication group." +vpcSecurityGroupIDs: + - "" + +#@schema/title "InstanceClassName" +#@schema/desc "The name of the instance class we want to use for binding secrets." +instanceClassName: aws-elasticache +#@schema/title "Service Instance Labels" +#@schema/desc "A set of labels which will be applied to the claimable secret." +#@schema/default {} +#@schema/type any=True +serviceInstanceLabels: + +#@schema/title "EngineVersion" +#@schema/desc "The version of the Redis engine to use. Available versions can be obtained running `aws cdescribe-cache-engine-versions --engine redis`." +engineVersion: "6.2" + +#@schema/title "ReplicasPerNodeGroup" +#@schema/desc "Number of replicas per node group. Allowed values are from 0 to 5." +replicasPerNodeGroup: 0 + +#@schema/title "Tags" +#@schema/desc "Tags to attach to all the resources." +#@schema/default [] +tags: + - + #@schema/title "Key" + #@schema/desc "The name of the tag." + key: "" + #@schema/title "Value" + #@schema/desc "The value of the tag." + value: "" diff --git a/packages/aws/carvel/elasticache/config/01-replication-group.ytt.yml b/packages/aws/carvel/elasticache/config/01-replication-group.ytt.yml new file mode 100644 index 0000000..c393e0c --- /dev/null +++ b/packages/aws/carvel/elasticache/config/01-replication-group.ytt.yml @@ -0,0 +1,221 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:assert", "assert") +#@ load("@ytt:template", "template") + +#@ def get_id(x): return "{}-{}".format(data.values.name, x) + +#@ def tags(resource_tags=[]): +#@ return [ x for x in (data.values.tags + resource_tags) if x["key"] != "" and x["value"] != "" ] +#@ end + +#@ users = { +#@ "reader": { "access_string": "on ~* -@all +@read" }, +#@ "writer": { "access_string": "on ~* +@all" }, +#@ } + +#! only Redis is currently supported +#@ engine = "redis" + +#@ replicationGroupName = data.values.name + +#@ service_account_name = "{}-{}".format(data.values.name, "elasticache-reader") +#@ role_name = service_account_name + +#! a user with name "default" must be present in every usergroup +#! the elasticache "default" user has all permissions and no password -> unsuitable +#! this creates a new "default" user with no permissions +#@ default_user = { "id": get_id("default"), "access_string": "off ~* -@all" } + +#@ mandatory_fields = [ "name", "namespace", "cacheSubnetGroupName", "cacheNodeType", "vpcSecurityGroupIDs" ] +#@ missing_fields = [] +#@ for field in mandatory_fields: +#@ if len(data.values[field]) == 0: +#@ missing_fields.append(field) +#@ end +#@ end + +#!@ if len(missing_fields) > 0: +#!@ assert.fail("Missing values for mandatory fields '{}'".format(missing_fields)) +#!@ end + +#@ if data.values.createCacheSubnetGroup and len(data.values.subnetIDs) == 0: +#@ assert.fail("SubnetIDs not provided for CacheSubnetGroup creation") +#@ end + +#@ if data.values.createNamespace and data.values.namespace: +--- +apiVersion: v1 +kind: Namespace +metadata: + name: #@ data.values.namespace +#@ end +--- +apiVersion: elasticache.services.k8s.aws/v1alpha1 +kind: User +metadata: + name: #@ default_user["id"] + namespace: #@ data.values.namespace +spec: + accessString: #@ default_user["access_string"] + noPasswordRequired: true + engine: #@ engine + userID: #@ default_user["id"] + userName: default + tags: #@ tags() + +#@ for user_name in users: +#@ user_id = get_id(user_name) +#@ user_creds = "{}-{}".format(user_id,"creds") +#@ users[user_name].update({ "id": user_id, "creds": user_creds }) +--- +apiVersion: secretgen.k14s.io/v1alpha1 +kind: Password +metadata: + name: #@ user_creds + namespace: #@ data.values.namespace +spec: + length: 128 + secretTemplate: + type: Opaque + stringData: + password: $(value) +--- +apiVersion: secretgen.carvel.dev/v1alpha1 +kind: SecretTemplate +metadata: + name: #@ "{}-bindable".format(user_creds) + namespace: #@ data.values.namespace +spec: + serviceAccountName: #@ service_account_name + inputResources: + - name: replicationGroup + ref: + apiVersion: elasticache.services.k8s.aws/v1alpha1 + kind: ReplicationGroup + name: #@ data.values.name + - name: creds + ref: + apiVersion: v1 + kind: Secret + name: #@ user_creds + template: + metadata: + labels: + services.apps.tanzu.vmware.com/class: #@ data.values.instanceClassName + _serviceInstanceLabels: #@ template.replace(data.values.serviceInstanceLabels) + type: servicebinding.io/redis + stringData: + type: redis + username: #@ user_name + ssl: "true" + host: $(.replicationGroup.status.nodeGroups[0].primaryEndpoint.address) + port: $(.replicationGroup.status.nodeGroups[0].primaryEndpoint.port) + data: + password: "$(.creds.data.password)" +--- +apiVersion: elasticache.services.k8s.aws/v1alpha1 +kind: User +metadata: + name: #@ user_id + namespace: #@ data.values.namespace +spec: + accessString: #@ users[user_name]["access_string"] + engine: #@ engine + passwords: + - name: #@ user_creds + key: password + namespace: #@ data.values.namespace + userID: #@ user_id + userName: #@ user_name + tags: #@ tags() +#@ end +--- +apiVersion: elasticache.services.k8s.aws/v1alpha1 +kind: UserGroup +metadata: + name: #@ data.values.name + namespace: #@ data.values.namespace +spec: + engine: #@ engine + userGroupID: #@ data.values.name + userIDs: #@ [ default_user["id"] ] + [ get_id(u) for u in users ] + tags: #@ tags() + +#@ if data.values.createCacheSubnetGroup: +--- +apiVersion: elasticache.services.k8s.aws/v1alpha1 +kind: CacheSubnetGroup +metadata: + name: #@ data.values.cacheSubnetGroupName + namespace: #@ data.values.namespace +spec: + cacheSubnetGroupDescription: #@ "CacheSubnetGroup for {}".format(data.values.name) + cacheSubnetGroupName: #@ data.values.cacheSubnetGroupName + subnetIDs: #@ data.values.subnetIDs +#@ end +--- +apiVersion: elasticache.services.k8s.aws/v1alpha1 +kind: ReplicationGroup +metadata: + name: #@ replicationGroupName + namespace: #@ data.values.namespace +spec: + description: #@ "A {} service instance".format(engine) + engine: #@ engine + engineVersion: #@ data.values.engineVersion + replicationGroupID: #@ replicationGroupName + cacheNodeType: #@ data.values.cacheNodeType + cacheSubnetGroupName: #@ data.values.cacheSubnetGroupName + securityGroupIDs: #@ data.values.vpcSecurityGroupIDs + atRestEncryptionEnabled: true + transitEncryptionEnabled: true + replicasPerNodeGroup: #@ data.values.replicasPerNodeGroup + userGroupIDs: + - #@ data.values.name + tags: #@ tags() +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: #@ service_account_name + namespace: #@ data.values.namespace +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: #@ role_name + namespace: #@ data.values.namespace +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + resourceNames: #@ [ users[u]["creds"] for u in users ] +- apiGroups: + - elasticache.services.k8s.aws + resources: + - replicationgroups + verbs: + - get + - list + - watch + resourceNames: + - #@ data.values.name +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: #@ role_name + namespace: #@ data.values.namespace +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: #@ role_name +subjects: +- kind: ServiceAccount + name: #@ service_account_name + namespace: #@ data.values.namespace diff --git a/packages/aws/carvel/elasticache/config/99-kapp-config.yml b/packages/aws/carvel/elasticache/config/99-kapp-config.yml new file mode 100644 index 0000000..bf40b13 --- /dev/null +++ b/packages/aws/carvel/elasticache/config/99-kapp-config.yml @@ -0,0 +1,43 @@ +--- +apiVersion: kapp.k14s.io/v1alpha1 +kind: Config + +waitRules: +- supportsObservedGeneration: false + conditionMatchers: + - type: ACK.ResourceSynced + status: "True" + success: true + - type: ACK.Terminal + status: "True" + failure: true + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: elasticache.services.k8s.aws/v1alpha1, kind: ReplicationGroup} + - apiVersionKindMatcher: {apiVersion: elasticache.services.k8s.aws/v1alpha1, kind: User} + - apiVersionKindMatcher: {apiVersion: elasticache.services.k8s.aws/v1alpha1, kind: UserGroup} + +- supportsObservedGeneration: false + conditionMatchers: + - type: ReconcileSucceeded + status: "True" + success: true + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: secretgen.carvel.dev/v1alpha1, kind: SecretTemplate} + +rebaseRules: +- paths: + - [spec, atRestEncryptionEnabled] + - [spec, cacheNodeType] + - [spec, cacheSubnetGroupName] + - [spec, description] + - [spec, engine] + - [spec, replicationGroupID] + - [spec, securityGroupIDs] + - [spec, snapshotRetentionLimit] + - [spec, snapshotWindow] + - [spec, transitEncryptionEnabled] + - [spec, userGroupIDs] + type: copy + sources: [new, existing] + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: elasticache.services.k8s.aws/v1alpha1, kind: ReplicationGroup} diff --git a/packages/aws/carvel/elasticache/package-metadata.yml b/packages/aws/carvel/elasticache/package-metadata.yml new file mode 100644 index 0000000..73eee60 --- /dev/null +++ b/packages/aws/carvel/elasticache/package-metadata.yml @@ -0,0 +1,17 @@ +apiVersion: data.packaging.carvel.dev/v1alpha1 +kind: PackageMetadata +metadata: + name: elasticache.aws.ref.services.apps.tanzu.vmware.com +spec: + categories: + - services + displayName: AWS Elasticache instances + longDescription: | + Helps you install an AWS Elasticache for Redis using ACK (https://github.com/aws-controllers-k8s/elasticache-controller). + You can follow the guide at https://vmware-tanzu.github.io/tanzu-application-platform-reference-service-packages/usecases/aws/prerequisites/ack/ + to quickly find relevant information about how to install and configure ACK for Elasticache. + maintainers: + - name: The TAP Reference Packages Team + providerName: VMware + shortDescription: AWS Elasticache with ACK controller + supportDescription: Not supported - provided as reference package only diff --git a/packages/azure/carvel/psql/config/00-schema.yml b/packages/azure/carvel/psql/config/00-schema.yml new file mode 100644 index 0000000..351c32b --- /dev/null +++ b/packages/azure/carvel/psql/config/00-schema.yml @@ -0,0 +1,140 @@ +#@data/values-schema + +--- +#@schema/title "ASO Controller Namespace" +#@schema/desc "The Namespace where the Azure ASO controller is installed that should own this Azure RM resource" +aso_controller_namespace: "azureserviceoperator-system" + +#@schema/title "ResourceName" +#@schema/desc "Name for the resources" +name: "aso-psql" + +#@schema/title "ResourceNamespace" +#@schema/desc "Kubernetes namespace where the Azure resources will be created" +namespace: "" + +#@schema/title "Create namespace flag" +#@schema/desc "Whether to create the namespace for the resources or not" +create_namespace: False + +#@schema/title "ResourceGroup" +#@schema/desc "Azure ResourceGroup for the servers/database resources" +resource_group: + #@schema/title "Name" + #@schema/desc "Azure ResourceGroup name" + name: "aso-psql" + #@schema/title "UseExisting" + #@schema/desc "Whether to use the existing Azure resource group or not" + use_existing: False + #@schema/title "Tags" + #@schema/desc "Tags to attach to the object" + #@schema/default [] + tags: + - + #@schema/title "Key" + #@schema/desc "The name of the tag" + key: "" + #@schema/title "Value" + #@schema/desc "The value of the tag" + value: "" + + +#@schema/title "Location" +#@schema/desc "Location where the resources will be created" +location: "" + +#@schema/title "FlexibleServer" +#@schema/desc "FlexibleServer instance that will be created" +server: + + #@schema/title "Name" + #@schema/desc "Flexible Server name. It must be unique across all Azure postgres database instances. Only lowercase letters, digits and hyphens are allowed." + name: "" + + #@schema/title "Version" + #@schema/desc "PostgreSQL version to deploy (only 11, 12 and 13 are currently supported" + version: "13" + + #@schema/title "AdministratorName" + #@schema/desc "Username for the administrator user. It cannot be 'azure_superuser', 'azuresu', 'azure_pg_admin', 'sa', 'admin', 'administrator', 'root', 'guest', 'dbmanager', 'loginmanager', 'dbo', 'information_schema', 'sys', 'db_accessadmin', 'db_backupoperator', 'db_datareader', 'db_datawriter', 'db_ddladmin', 'db_denydatareader', 'db_denydatawriter', 'db_owner', 'db_securityadmin', 'public'." + administrator_name: "myadmin" + + #@schema/title "InstanceType" + #@schema/desc "The type of the requested instance (follows the convention Standard_{VM name})" + instance_type: "Standard_D2s_v3" + + #@schema/title "InstanceTier" + #@schema/desc "The tier of the requested instance (allowed: 'Burstable', 'GeneralPurpose' or 'Memory Optimized')" + instance_tier: "GeneralPurpose" + + #@schema/title "InstanceStorageSizeGB" + #@schema/desc "The storage size for the instance in GB (allowed: from 32 to 16384)" + instance_storage_size_gb: 128 + + #@schema/title "Tags" + #@schema/desc "Tags to attach to the object" + #@schema/default [] + tags: + - + #@schema/title "Key" + #@schema/desc "The name of the tag" + key: "" + #@schema/title "Value" + #@schema/desc "The value of the tag" + value: "" + +#@schema/title "Database" +#@schema/desc "The database that will be created." +database: + + #@schema/title "Name" + #@schema/desc "Name of the database" + name: "" + + #@schema/title "Tags" + #@schema/desc "Tags to attach to the object" + #@schema/default [] + tags: + - + #@schema/title "Key" + #@schema/desc "The name of the tag" + key: "" + #@schema/title "Value" + #@schema/desc "The value of the tag" + value: "" + +#@schema/title "FirewallRules" +#@schema/desc "List of firewall rules for exposing the Flexible Server. '0.0.0.0' for both startIpAddress and endIpAddress means it will be available from Azure (not the whole public Internet). Must be IPv4 format." +#@schema/default [] +firewall_rules: + + - + #@schema/title "StartIpAddress" + #@schema/desc "The starting IP address of the range" + startIpAddress: "" + #@schema/title "EndIpAddress" + #@schema/desc "The ending IP address of the range" + endIpAddress: "" + #@schema/title "Tags" + #@schema/desc "Tags to attach to the object" + #@schema/default [] + tags: + - + #@schema/title "Key" + #@schema/desc "The name of the tag" + key: "" + #@schema/title "Value" + #@schema/desc "The value of the tag" + value: "" + +#@schema/title "GlobalTags" +#@schema/desc "Tags to attach to all the resources" +#@schema/default [] +global_tags: + - + #@schema/title "Key" + #@schema/desc "The name of the tag" + key: "" + #@schema/title "Value" + #@schema/desc "The value of the tag" + value: "" diff --git a/packages/azure/carvel/psql/config/01-flexible-server.ytt.yml b/packages/azure/carvel/psql/config/01-flexible-server.ytt.yml new file mode 100644 index 0000000..cb0445e --- /dev/null +++ b/packages/azure/carvel/psql/config/01-flexible-server.ytt.yml @@ -0,0 +1,246 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:assert", "assert") +#@ load("@ytt:regexp", "regexp") +#@ load("@ytt:ip", "ip") + +#@ def tags(resource_tags=[]): +#@ tags_list = [] +#@ tags_list.extend(data.values.global_tags) +#@ tags_list.extend(resource_tags) +#@ tags_map = { x["key"]: x["value"] for x in tags_list if x["key"] != "" and x["value"] != "" } +#@ return(tags_map) +#@ end + +#@ flexibleServerK8SName = data.values.name +#@ dbSecretName = data.values.name +#@ resourceGroupK8SName = data.values.name +#@ serviceAccountName = data.values.name + "-reader" +#@ serviceAccountNamespace = data.values.namespace +#@ roleName = data.values.name + "-reading" + +#@ firewallRules = data.values.firewall_rules +#@ if len(firewallRules) == 0: +#@ firewallRules = [{"startIpAddress": "0.0.0.0", "endIpAddress": "0.0.0.0", "tags": []}] +#@ end + +#@ mandatory_fields = [ "name", "location", "aso_controller_namespace" ] +#@ missing_fields = [] +#@ for field in mandatory_fields: +#@ if len(data.values[field]) == 0: +#@ missing_fields.append(field) +#@ end +#@ end +#@ if data.values.resource_group.name == "": +#@ missing_fields.append("resource_group.name") +#@ end + +#@ databaseName = "name" in data.values.database and data.values.database.name or data.values.name + +#!@ if len(missing_fields) > 0: +#!@ assert.fail("Missing values for mandatory fields '{}'".format(missing_fields)) +#!@ end +#@ if not regexp.match("^Standard_", data.values.server.instance_type): +#@ assert.fail("'instance_type' must follow the convention Standard_(VMname) (was: '{}')".format(data.values.server.instance_type)) +#@ end +#@ if data.values.server.instance_tier not in ["Burstable", "GeneralPurpose", "Memory Optimized"]: +#@ assert.fail("'instance_tier' must be in [\"Burstable\", \"GeneralPurpose\", \"Memory Optimized\"] (was: '{}')".format(data.values.server.instance_tier)) +#@ end +#@ if data.values.server.instance_storage_size_gb < 32 or data.values.server.instance_storage_size_gb > 16384: +#@ assert.fail("'instance_storage_size_gb' must be within 32 and 16384 (was: {})".format(data.values.server.instance_storage_size_gb)) +#@ end + +#@ if data.values.create_namespace and data.values.namespace: +--- +apiVersion: v1 +kind: Namespace +metadata: + name: #@ data.values.namespace +#@ end +--- +apiVersion: secretgen.k14s.io/v1alpha1 +kind: Password +metadata: + name: #@ dbSecretName +#@ if/end data.values.namespace: + namespace: #@ data.values.namespace +spec: + length: 64 + secretTemplate: + type: Opaque + stringData: + password: $(value) +--- +apiVersion: resources.azure.com/v1beta20200601 +kind: ResourceGroup +metadata: + name: #@ resourceGroupK8SName +#@ if/end data.values.namespace: + namespace: #@ data.values.namespace + annotations: + serviceoperator.azure.com/operator-namespace: #@ data.values.aso_controller_namespace +#@ if/end data.values.resource_group.use_existing: + serviceoperator.azure.com/reconcile-policy: skip + labels: + kapp.k14s.io/noop: "" +spec: + azureName: #@ data.values.resource_group.name + location: #@ data.values.location + tags: #@ tags(data.values.resource_group.tags) +--- +apiVersion: dbforpostgresql.azure.com/v1beta20210601 +kind: FlexibleServersDatabase +metadata: + name: #@ data.values.name +#@ if/end data.values.namespace: + namespace: #@ data.values.namespace +spec: + azureName: #@ databaseName + owner: + name: #@ flexibleServerK8SName + charset: utf8 + tags: #@ tags(data.values.database.tags) + +#@ fwRuleCounter = 0 +#@ for rule in firewallRules: +#@ startAddr = ip.parse_addr(rule["startIpAddress"]) +#@ endAddr = ip.parse_addr(rule["endIpAddress"]) +--- +apiVersion: dbforpostgresql.azure.com/v1beta20210601 +kind: FlexibleServersFirewallRule +metadata: + name: #@ data.values.name + "-" + str(fwRuleCounter) +#@ if/end data.values.namespace: + namespace: #@ data.values.namespace +spec: + owner: + name: #@ flexibleServerK8SName + startIpAddress: #@ startAddr.string() + endIpAddress: #@ endAddr.string() + tags: #@ tags(rule["tags"]) +#@ fwRuleCounter += 1 +#@ end +--- +apiVersion: dbforpostgresql.azure.com/v1beta20210601 +kind: FlexibleServer +metadata: + name: #@ flexibleServerK8SName +#@ if/end data.values.namespace: + namespace: #@ data.values.namespace +spec: + location: #@ data.values.location + azureName: #@ data.values.server.name or flexibleServerK8SName + owner: + name: #@ resourceGroupK8SName + version: #@ data.values.server.version + sku: + name: #@ data.values.server.instance_type + tier: #@ data.values.server.instance_tier + administratorLogin: #@ data.values.server.administrator_name + administratorLoginPassword: + name: #@ dbSecretName + key: password + storage: + storageSizeGB: #@ data.values.server.instance_storage_size_gb + tags: #@ tags(data.values.server.tags) +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: #@ serviceAccountName +#@ if/end serviceAccountNamespace: + namespace: #@ serviceAccountNamespace +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: #@ roleName +#@ if/end data.values.namespace: + namespace: #@ data.values.namespace +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + resourceNames: + - #@ data.values.name +- apiGroups: + - dbforpostgresql.azure.com + resources: + - flexibleservers + - flexibleserversdatabases + verbs: + - get + - list + - watch + resourceNames: + - #@ data.values.name +- apiGroups: + - resources.azure.com + resources: + - resourcegroups + verbs: + - get + - list + - watch + resourceNames: + - #@ resourceGroupK8SName +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: #@ roleName +#@ if/end data.values.namespace: + namespace: #@ data.values.namespace +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: #@ roleName +subjects: +- kind: ServiceAccount + name: #@ serviceAccountName +#@ if/end serviceAccountNamespace: + namespace: #@ serviceAccountNamespace +--- +apiVersion: secretgen.carvel.dev/v1alpha1 +kind: SecretTemplate +metadata: + name: #@ data.values.name + "-bindable" +#@ if/end data.values.namespace: + namespace: #@ data.values.namespace +spec: + serviceAccountName: #@ serviceAccountName + inputResources: + - name: server + ref: + apiVersion: dbforpostgresql.azure.com/v1alpha1api20210601 + kind: FlexibleServer + name: #@ flexibleServerK8SName + - name: db + ref: + apiVersion: dbforpostgresql.azure.com/v1alpha1api20210601 + kind: FlexibleServersDatabase + name: #@ data.values.name + - name: creds + ref: + apiVersion: v1 + kind: Secret + name: "$(.server.spec.administratorLoginPassword.name)" + template: + metadata: + labels: + app.kubernetes.io/component: #@ data.values.name + app.kubernetes.io/instance: "$(.server.metadata.name)" + services.apps.tanzu.vmware.com/class: azure-postgres + type: postgresql + stringData: + type: postgresql + port: "5432" + database: "$(.db.status.name)" + host: "$(.server.status.fullyQualifiedDomainName)" + username: "$(.server.status.administratorLogin)" + data: + password: "$(.creds.data.password)" diff --git a/packages/azure/carvel/psql/config/99-kapp-config.yml b/packages/azure/carvel/psql/config/99-kapp-config.yml new file mode 100644 index 0000000..95bd6db --- /dev/null +++ b/packages/azure/carvel/psql/config/99-kapp-config.yml @@ -0,0 +1,53 @@ +apiVersion: kapp.k14s.io/v1alpha1 +kind: Config +minimumRequiredVersion: 0.48.0 +waitRules: +- supportsObservedGeneration: false + ytt: + funcContractV1: + resource.star: | + def is_done(resource): + kind = resource.kind + name = resource.metadata.name + + if not "status" in resource or not "conditions" in resource.status: + return {"done": False, "successful": False, "message": "Resource " + kind + "/" + name + " waiting for status"} + end + + condition = resource.status.conditions[0] + reason = "" + if "reason" in condition: + reason = condition.reason + end + + if repr(condition.type) == repr("Ready") and repr(reason) == repr("Succeeded"): + return {"done": True, "successful": True, "message": "Resource " + kind + "/" + name + " created"} + elif repr(condition.type) == repr("Ready") and repr(condition.status) == repr("False") and repr(condition.severity) == repr("Error"): + return {"done": True, "successful": False, "message": "Resource " + kind + "/" + name + " creation failed: " + reason} + else: + return {"done": False, "successful": False, "message": "Resource " + kind + "/" + name + " still being reconciled: " + reason} + end + end + resourceMatchers: &flexibleServerResources + - apiVersionKindMatcher: {apiVersion: resources.azure.com/v1beta20200601, kind: ResourceGroup} + - apiVersionKindMatcher: {apiVersion: dbforpostgresql.azure.com/v1beta20210601, kind: FlexibleServer} + - apiVersionKindMatcher: {apiVersion: dbforpostgresql.azure.com/v1beta20210601, kind: FlexibleServersDatabase} + - apiVersionKindMatcher: {apiVersion: dbforpostgresql.azure.com/v1beta20210601, kind: FlexibleServersFirewallRule} + +- supportsObservedGeneration: false + conditionMatchers: + - type: ReconcileSucceeded + status: "True" + success: true + resourceMatchers: + - apiVersionKindMatcher: {apiVersion: secretgen.carvel.dev/v1alpha1, kind: SecretTemplate} + +rebaseRules: +- paths: + - [metadata, annotations, serviceoperator.azure.com/operator-namespace] + - [metadata, annotations, serviceoperator.azure.com/resource-id] + - [metadata, annotations, serviceoperator.azure.com/poller-resume-id] + - [metadata, annotations, serviceoperator.azure.com/poller-resume-token] + type: copy + sources: [new, existing] + resourceMatchers: *flexibleServerResources \ No newline at end of file diff --git a/repository/packages/azure/psql/metadata.yml b/packages/azure/carvel/psql/package-metadata.yml similarity index 58% rename from repository/packages/azure/psql/metadata.yml rename to packages/azure/carvel/psql/package-metadata.yml index cd82ec2..632b120 100644 --- a/repository/packages/azure/psql/metadata.yml +++ b/packages/azure/carvel/psql/package-metadata.yml @@ -1,16 +1,15 @@ apiVersion: data.packaging.carvel.dev/v1alpha1 kind: PackageMetadata metadata: - creationTimestamp: null - name: psql.azure.references.services.apps.tanzu.vmware.com + name: psql.azure.ref.services.apps.tanzu.vmware.com spec: categories: - services - displayName: Azure Flexible Server for Postgresql + displayName: Azure Flexible Server for PostgreSQL longDescription: | - Helps you install an Azure Flexible Server for Postgresql which can be bound to your application via a ServiceBinding. + Helps you install an Azure Flexible Server for PostgreSQL which can be bound to your application via a ServiceBinding. maintainers: - name: The TAP Reference Packages Team providerName: VMware - shortDescription: dbforpostgresql.azure.com + shortDescription: Azure PostgreSQL with ASO supportDescription: Not supported - provided as reference package only diff --git a/repository/.imgpkg/images.yml b/repository/.imgpkg/images.yml deleted file mode 100644 index 5eaecdb..0000000 --- a/repository/.imgpkg/images.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -apiVersion: imgpkg.carvel.dev/v1alpha1 -images: -- annotations: - kbld.carvel.dev/id: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/elasticache.aws.references.services.apps.tanzu.vmware.com@sha256:cd0ed7ad049c51116e57d604c62214344d1f53b18c5a9c189329839a80401caa - kbld.carvel.dev/origins: | - - preresolved: - url: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/elasticache.aws.references.services.apps.tanzu.vmware.com@sha256:cd0ed7ad049c51116e57d604c62214344d1f53b18c5a9c189329839a80401caa - image: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/elasticache.aws.references.services.apps.tanzu.vmware.com@sha256:cd0ed7ad049c51116e57d604c62214344d1f53b18c5a9c189329839a80401caa -- annotations: - kbld.carvel.dev/id: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/psql.aws.references.services.apps.tanzu.vmware.com@sha256:01481af67e9be08c466e7d98eb1e2e0e904a4965d289d793c41d316aeef59332 - kbld.carvel.dev/origins: | - - preresolved: - url: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/psql.aws.references.services.apps.tanzu.vmware.com@sha256:01481af67e9be08c466e7d98eb1e2e0e904a4965d289d793c41d316aeef59332 - image: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/psql.aws.references.services.apps.tanzu.vmware.com@sha256:01481af67e9be08c466e7d98eb1e2e0e904a4965d289d793c41d316aeef59332 -- annotations: - kbld.carvel.dev/id: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/psql.azure.references.services.apps.tanzu.vmware.com@sha256:c8342f48fce1c27d84a51efb923eefee077ea4b457a3df109598363068641dd9 - kbld.carvel.dev/origins: | - - preresolved: - url: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/psql.azure.references.services.apps.tanzu.vmware.com@sha256:c8342f48fce1c27d84a51efb923eefee077ea4b457a3df109598363068641dd9 - image: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/psql.azure.references.services.apps.tanzu.vmware.com@sha256:c8342f48fce1c27d84a51efb923eefee077ea4b457a3df109598363068641dd9 -- annotations: - kbld.carvel.dev/id: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/psql.google.references.services.apps.tanzu.vmware.com@sha256:9fc1a2767adac504e06d6eb8c9b2cbecc887ad605e3652401f26dc101abd1c6d - kbld.carvel.dev/origins: | - - preresolved: - url: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/psql.google.references.services.apps.tanzu.vmware.com@sha256:9fc1a2767adac504e06d6eb8c9b2cbecc887ad605e3652401f26dc101abd1c6d - image: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/psql.google.references.services.apps.tanzu.vmware.com@sha256:9fc1a2767adac504e06d6eb8c9b2cbecc887ad605e3652401f26dc101abd1c6d -kind: ImagesLock diff --git a/repository/packages/amazon/elasticache/metadata.yml b/repository/packages/amazon/elasticache/metadata.yml deleted file mode 100644 index e2f898b..0000000 --- a/repository/packages/amazon/elasticache/metadata.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: data.packaging.carvel.dev/v1alpha1 -kind: PackageMetadata -metadata: - creationTimestamp: null - name: elasticache.aws.references.services.apps.tanzu.vmware.com -spec: - categories: - - services - displayName: AWS Elasticache instances - longDescription: | - Helps you install an AWS Elasticache for Redis using ACK - maintainers: - - name: The TAP Reference Packages Team - providerName: VMware - shortDescription: elasticache.services.k8s.aws - supportDescription: Not supported - provided as reference package only diff --git a/repository/packages/amazon/elasticache/package.yml b/repository/packages/amazon/elasticache/package.yml deleted file mode 100644 index 16eed5b..0000000 --- a/repository/packages/amazon/elasticache/package.yml +++ /dev/null @@ -1,122 +0,0 @@ ---- -apiVersion: data.packaging.carvel.dev/v1alpha1 -kind: Package -metadata: - annotations: - kbld.k14s.io/images: | - - origins: - - resolved: - tag: 0.0.1-alpha - url: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/elasticache.aws.references.services.apps.tanzu.vmware.com:0.0.1-alpha - url: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/elasticache.aws.references.services.apps.tanzu.vmware.com@sha256:cd0ed7ad049c51116e57d604c62214344d1f53b18c5a9c189329839a80401caa - creationTimestamp: null - name: elasticache.aws.references.services.apps.tanzu.vmware.com.0.0.1-alpha -spec: - refName: elasticache.aws.references.services.apps.tanzu.vmware.com - releasedAt: "2022-10-19T09:41:52Z" - template: - spec: - deploy: - - kapp: {} - fetch: - - imgpkgBundle: - image: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/elasticache.aws.references.services.apps.tanzu.vmware.com@sha256:cd0ed7ad049c51116e57d604c62214344d1f53b18c5a9c189329839a80401caa - template: - - ytt: - paths: - - config - - kbld: - paths: - - '-' - - .imgpkg/images.yml - valuesSchema: - openAPIv3: - additionalProperties: false - properties: - name: - title: Name - type: string - description: Name of the Elasticache instance. - default: "" - namespace: - title: Namespace - type: string - description: Namespace to deploy the kubernetes resources to. - default: "" - createNamespace: - title: CreateNamespace - type: boolean - description: Whether to create the namespace for the resources or not. - default: false - cacheSubnetGroupName: - title: CacheSubnetGroupName - type: string - description: Name of the cache subnet group to create the replication group in. It must be pre-created if the createCacheSubnetGroup parameter is set to false. - default: "" - createCacheSubnetGroup: - title: CreateCacheSubnetGroup - type: boolean - description: Whether to create the CacheSubnetGroup or not. - default: false - subnetIDs: - title: SubnetIDs - type: array - description: The list of subnets to create the CacheSubnetGroup for. Mandatory if the createCacheSubnetGroup parameter is set to true. - items: - type: string - default: "" - default: [] - cacheNodeType: - title: CacheNodeType - type: string - description: Type of nodes used for the replication group. - default: cache.t2.micro - vpcSecurityGroupIDs: - title: VpcSecurityGroupIDs - type: array - description: The list of security groups to associate to the replication group. - items: - type: string - default: "" - default: [] - instanceClassName: - title: InstanceClassName - type: string - description: The name of the instance class we want to use for binding secrets. - default: aws-elasticache - serviceInstanceLabels: - title: Service Instance Labels - nullable: true - description: A set of labels which will be applied to the claimable secret. - default: {} - engineVersion: - title: EngineVersion - type: string - description: The version of the Redis engine to use. Available versions can be obtained running `aws cdescribe-cache-engine-versions --engine redis`. - default: "6.2" - replicasPerNodeGroup: - title: ReplicasPerNodeGroup - type: integer - description: Number of replicas per node group. Allowed values are from 0 to 5. - default: 0 - tags: - title: Tags - type: array - description: Tags to attach to all the resources. - items: - type: object - additionalProperties: false - properties: - key: - title: Key - type: string - description: The name of the tag. - default: "" - value: - title: Value - type: string - description: The value of the tag. - default: "" - default: [] - type: object - version: 0.0.1-alpha diff --git a/repository/packages/amazon/rds/meta.yaml b/repository/packages/amazon/rds/meta.yaml deleted file mode 100644 index 6c64fac..0000000 --- a/repository/packages/amazon/rds/meta.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: data.packaging.carvel.dev/v1alpha1 -kind: PackageMetadata -metadata: - name: psql.aws.references.services.apps.tanzu.vmware.com -spec: - categories: - - services - displayName: Amazon RDS instances - longDescription: | - Install Amazon RDS instances - maintainers: - - name: The Services Toolkit team - providerName: VMware - shortDescription: rds - supportDescription: Not supported - provided as reference package only diff --git a/repository/packages/amazon/rds/package.yaml b/repository/packages/amazon/rds/package.yaml deleted file mode 100644 index 3902927..0000000 --- a/repository/packages/amazon/rds/package.yaml +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: data.packaging.carvel.dev/v1alpha1 -kind: Package -metadata: - name: psql.aws.references.services.apps.tanzu.vmware.com.0.0.1-alpha -spec: - refName: psql.aws.references.services.apps.tanzu.vmware.com - version: 0.0.1-alpha - releasedAt: "2021-12-07T15:58:50+00:00" - releaseNotes: https://docs.vmware.com/en/Services-Toolkit-for-VMware-Tanzu/0.7/services-toolkit-0-7/GUID-overview.html - valuesSchema: - openAPIv3: - type: object - additionalProperties: false - properties: - name: - type: string - default: "" - namespace: - type: string - default: default - database: - type: string - default: postgres - vpcSecurityGroupIDs: - type: array - items: - type: string - default: "" - default: [] - dbSubnetGroupName: - type: string - default: "" - engine: - type: object - additionalProperties: - allow: false - properties: - version: - type: string - description: The PostgresVersion to use. - enum: - - "14" - template: - spec: - fetch: - - imgpkgBundle: - image: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/psql.aws.references.services.apps.tanzu.vmware.com@sha256:01481af67e9be08c466e7d98eb1e2e0e904a4965d289d793c41d316aeef59332 - template: - - ytt: - paths: - - config/ - - kbld: - paths: - - "-" - - ".imgpkg/images.yml" - deploy: - - kapp: {} diff --git a/repository/packages/azure/psql/package.yml b/repository/packages/azure/psql/package.yml deleted file mode 100644 index 0786b36..0000000 --- a/repository/packages/azure/psql/package.yml +++ /dev/null @@ -1,246 +0,0 @@ ---- -apiVersion: data.packaging.carvel.dev/v1alpha1 -kind: Package -metadata: - creationTimestamp: null - name: psql.azure.references.services.apps.tanzu.vmware.com.0.0.1-alpha -spec: - refName: psql.azure.references.services.apps.tanzu.vmware.com - releasedAt: "2022-09-26T10:17:49Z" - template: - spec: - deploy: - - kapp: {} - fetch: - - imgpkgBundle: - image: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/psql.azure.references.services.apps.tanzu.vmware.com@sha256:c8342f48fce1c27d84a51efb923eefee077ea4b457a3df109598363068641dd9 - template: - - ytt: - paths: - - config - - kbld: - paths: - - '-' - - .imgpkg/images.yml - valuesSchema: - openAPIv3: - additionalProperties: false - properties: - aso_controller_namespace: - default: azureserviceoperator-system - description: The Namespace where the Azure ASO controller is installed that - should own this Azure RM resource - title: ASO Controller Namespace - type: string - create_namespace: - default: false - description: Whether to create the namespace for the resources or not - title: Create namespace flag - type: boolean - database: - additionalProperties: false - description: The database that will be created. - properties: - name: - default: "" - description: Name of the database - title: Name - type: string - tags: - default: [] - description: Tags to attach to the object - items: - additionalProperties: false - properties: - key: - default: "" - description: The name of the tag - title: Key - type: string - value: - default: "" - description: The value of the tag - title: Value - type: string - type: object - title: Tags - type: array - title: Database - type: object - firewall_rules: - default: [] - description: List of firewall rules for exposing the Flexible Server. '0.0.0.0' - for both startIpAddress and endIpAddress means it will be available from - Azure (not the whole public Internet). Must be IPv4 format. - items: - additionalProperties: false - properties: - endIpAddress: - default: "" - description: The ending IP address of the range - title: EndIpAddress - type: string - startIpAddress: - default: "" - description: The starting IP address of the range - title: StartIpAddress - type: string - tags: - default: [] - description: Tags to attach to the object - items: - additionalProperties: false - properties: - key: - default: "" - description: The name of the tag - title: Key - type: string - value: - default: "" - description: The value of the tag - title: Value - type: string - type: object - title: Tags - type: array - type: object - title: FirewallRules - type: array - global_tags: - default: [] - description: Tags to attach to all the resources - items: - additionalProperties: false - properties: - key: - default: "" - description: The name of the tag - title: Key - type: string - value: - default: "" - description: The value of the tag - title: Value - type: string - type: object - title: GlobalTags - type: array - location: - default: "" - description: Location where the resources will be created - title: Location - type: string - name: - default: aso-psql - description: Name for the resources - title: ResourceName - type: string - namespace: - default: "" - description: Kubernetes namespace where the Azure resources will be created - title: ResourceNamespace - type: string - resource_group: - additionalProperties: false - description: Azure ResourceGroup for the servers/database resources - properties: - name: - default: aso-psql - description: Azure ResourceGroup name - title: Name - type: string - tags: - default: [] - description: Tags to attach to the object - items: - additionalProperties: false - properties: - key: - default: "" - description: The name of the tag - title: Key - type: string - value: - default: "" - description: The value of the tag - title: Value - type: string - type: object - title: Tags - type: array - use_existing: - default: false - description: Whether to use the existing Azure resource group or not - title: UseExisting - type: boolean - title: ResourceGroup - type: object - server: - additionalProperties: false - description: FlexibleServer instance that will be created - properties: - administrator_name: - default: myadmin - description: Username for the administrator user. It cannot be 'azure_superuser', - 'azuresu', 'azure_pg_admin', 'sa', 'admin', 'administrator', 'root', - 'guest', 'dbmanager', 'loginmanager', 'dbo', 'information_schema', - 'sys', 'db_accessadmin', 'db_backupoperator', 'db_datareader', 'db_datawriter', - 'db_ddladmin', 'db_denydatareader', 'db_denydatawriter', 'db_owner', - 'db_securityadmin', 'public'. - title: AdministratorName - type: string - instance_storage_size_gb: - default: 128 - description: 'The storage size for the instance in GB (allowed: from - 32 to 16384)' - title: InstanceStorageSizeGB - type: integer - instance_tier: - default: GeneralPurpose - description: 'The tier of the requested instance (allowed: ''Burstable'', - ''GeneralPurpose'' or ''Memory Optimized'')' - title: InstanceTier - type: string - instance_type: - default: Standard_D2s_v3 - description: The type of the requested instance (follows the convention - Standard_{VM name}) - title: InstanceType - type: string - name: - default: "" - description: Flexible Server name. It must be unique across all Azure - postgres database instances. Only lowercase letters, numbers and hyphens - are allowed. - title: Name - type: string - tags: - default: [] - description: Tags to attach to the object - items: - additionalProperties: false - properties: - key: - default: "" - description: The name of the tag - title: Key - type: string - value: - default: "" - description: The value of the tag - title: Value - type: string - type: object - title: Tags - type: array - version: - default: "13" - description: PostgreSQL version to deploy (only 11, 12 and 13 are currently - supported - title: Version - type: string - title: FlexibleServer - type: object - type: object - version: 0.0.1-alpha diff --git a/repository/packages/google/cloudsql/meta.yaml b/repository/packages/google/cloudsql/meta.yaml deleted file mode 100644 index 2a90874..0000000 --- a/repository/packages/google/cloudsql/meta.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: data.packaging.carvel.dev/v1alpha1 -kind: PackageMetadata -metadata: - name: psql.google.references.services.apps.tanzu.vmware.com -spec: - categories: - - services - displayName: Google Cloud SQL instance - longDescription: | - Install Google Cloud SQL instance - maintainers: - - name: The Services Toolkit team - providerName: VMware - shortDescription: cloudsql - supportDescription: Not supported - provided as reference package only diff --git a/repository/packages/google/cloudsql/package.yaml b/repository/packages/google/cloudsql/package.yaml deleted file mode 100644 index d006fe2..0000000 --- a/repository/packages/google/cloudsql/package.yaml +++ /dev/null @@ -1,101 +0,0 @@ -apiVersion: data.packaging.carvel.dev/v1alpha1 -kind: Package -metadata: - name: psql.google.references.services.apps.tanzu.vmware.com.0.0.1-alpha -spec: - refName: psql.google.references.services.apps.tanzu.vmware.com - version: 0.0.1-alpha - releasedAt: "2022-07-14T14:47:00+02:00" - releaseNotes: https://docs.vmware.com/en/Services-Toolkit-for-VMware-Tanzu/0.7/services-toolkit-0-7/GUID-overview.html - valuesSchema: - openAPIv3: - type: object - additionalProperties: false - properties: - name: - title: Service Instance Name - type: string - description: The name of the Cloud SQL instance and related objects - default: '' - namespace: - title: Service Instance Namespace - type: string - nullable: true - description: The namespace the service instance objects should be deployed into - default: null - version: - type: string - description: 'The database version of the Cloud SQL instance - - see: https://cloud.google.com/config-connector/docs/reference/resource-docs/sql/sqlinstance#:~:text=Fields-,databaseVersion,-Optional' - default: POSTGRES_14 - region: - title: Database Region - type: string - description: 'The region this Cloud SQL instance should be deployed into - - see: https://cloud.google.com/config-connector/docs/reference/resource-docs/sql/sqlinstance#:~:text=with%2Dobjects/namespaces/-,region,-Optional' - default: europe-west6 - tier: - title: Database Machine Type - type: string - description: 'The machine type for the Cloud SQL instance - - see: https://cloud.google.com/config-connector/docs/reference/resource-docs/sql/sqlinstance#:~:text=from%20your%20configuration.-,settings.tier,-Required' - default: db-g1-small - labels: - title: Labels - nullable: true - description: A set of labels which will be applied to all resources related to - this instance - x-example-description: Set custom labels on all objects - example: - mycorp.io/service-type: gcp-sql - mycorp.io/owner: me@mycorp.io - default: - app.kubernetes.io/component: cloudsql-postgres - serviceInstanceLabels: - title: Service Instance Labels - nullable: true - description: A set of labels which will be applied to the claimable secret - default: - services.apps.tanzu.vmware.com/class: cloudsql-postgres - allowedNetworks: - title: Allowed Networks - type: array - nullable: true - description: A list of CIDR ranges allowed to talk to this Cloud SQL instance - x-example-description: Allow one (named) network & one host - example: - - name: my-onprem-net - value: 11.22.33.44/24 - - value: 8.8.8.8/32 - items: - type: object - additionalProperties: false - properties: - name: - type: string - nullable: true - description: the name for the authorized/allowed network (optional) - default: null - value: - type: string - description: the IPv4 CIDR of the authorized/allowed network - default: '' - default: null - template: - spec: - fetch: - - imgpkgBundle: - image: ghcr.io/vmware-tanzu/tanzu-application-platform-reference-service-packages/psql.google.references.services.apps.tanzu.vmware.com@sha256:9fc1a2767adac504e06d6eb8c9b2cbecc887ad605e3652401f26dc101abd1c6d - template: - - ytt: - paths: - - config/ - - kbld: - paths: - - "-" - - ".imgpkg/images.yml" - deploy: - - kapp: {} diff --git a/scripts/carvel-azure-install-aso.sh b/scripts/carvel-azure-install-aso.sh new file mode 100755 index 0000000..f3f2bed --- /dev/null +++ b/scripts/carvel-azure-install-aso.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -euo pipefail + +CERT_MANAGER_MANIFEST="https://github.com/jetstack/cert-manager/releases/download/v1.8.2/cert-manager.yaml" +ASO_MANIFEST="https://github.com/Azure/azure-service-operator/releases/download/v2.0.0-beta.3/azureserviceoperator_v2.0.0-beta.3.yaml" +ASO_NAMESPACE="azureserviceoperator-system" + +echo ">> Install certmanager" +kubectl apply -f ${CERT_MANAGER_MANIFEST} +kubectl -n cert-manager wait --for=condition=Available=True deployments.apps cert-manager +kubectl -n cert-manager wait --for=condition=Available=True deployments.apps cert-manager-cainjector +kubectl -n cert-manager wait --for=condition=Available=True deployments.apps cert-manager-webhook +kubectl wait --for=condition=Available apiservices.apiregistration.k8s.io v1.cert-manager.io +kubectl wait --for=condition=Available apiservices.apiregistration.k8s.io v1.acme.cert-manager.io + +echo ">> Install Azure Service Operator" +kubectl create ns ${ASO_NAMESPACE} || true +cat <$VALUES +--- +name: ${NAME} +namespace: ${PACKAGE_NAMESPACE} +location: ${LOCATION} +aso_controller_namespace: azureserviceoperator-system +create_namespace: false + +server: + administrator_name: testadmin + +database: + name: testdb + +firewall_rules: + - startIpAddress: 0.0.0.0 + endIpAddress: 0.0.0.0 + - startIpAddress: ${PUBLIC_IP} + endIpAddress: ${PUBLIC_IP} + +resource_group: + use_existing: false + name: carvel-test-${NAME} +EOF +trap "rm ${VALUES}" EXIT + +# install package +kubectl create namespace ${PACKAGE_NAMESPACE} || true + +SA=${PACKAGE_METADATA_NAME} +INSTALL_NAME=${PACKAGE_METADATA_NAME} + +echo ">> Prepare RBAC" +ytt -f ./carvel-e2e-azure-psql/rbac.ytt.yml -v serviceAccount=${SA} -v namespace=${PACKAGE_NAMESPACE} | kubectl apply -f - + +echo ">> Install package" +kctrl package install -n ${PACKAGE_NAMESPACE} -i ${INSTALL_NAME} -p ${PACKAGE_METADATA_NAME} --version ${PACKAGE_VERSION} --values-file ${VALUES} --service-account-name ${SA} --wait=false + +RESTARTS_MAX=4 +RESTARTS_COUNT=0 +while [ $RESTARTS_COUNT -lt $RESTARTS_MAX ]; do + echo ">> Waiting for stack ${NAME} to reconcile..." + kubectl -n ${PACKAGE_NAMESPACE} wait --for=condition=ReconcileSucceeded --timeout=${TIMEOUT} packageinstalls.packaging.carvel.dev ${INSTALL_NAME} && AGAIN=0 || AGAIN=1 + if [ $AGAIN -eq 0 ]; then + RESTARTS_COUNT=$RESTARTS_MAX + else + # ASO needs to be kicked because it conflicts with kapp-controller for taking ownership of Azure resources + let RESTARTS_COUNT=$RESTARTS_COUNT+1 + kubectl -n azureserviceoperator-system rollout restart deployments.apps azureserviceoperator-controller-manager + fi +done + +# run test +SECRET_NAME="${NAME}-bindable" +./carvel-e2e-azure-psql/test.sh ${SECRET_NAME} ${APP_NAME} + +popd diff --git a/scripts/carvel-e2e-azure-psql/app-overlay.ytt.yml b/scripts/carvel-e2e-azure-psql/app-overlay.ytt.yml new file mode 100644 index 0000000..dcb05d8 --- /dev/null +++ b/scripts/carvel-e2e-azure-psql/app-overlay.ytt.yml @@ -0,0 +1,28 @@ +#@ load("@ytt:overlay", "overlay") +#@ load("@ytt:data", "data") + +#@ name = data.values.name + +#@overlay/match by=overlay.all, expects="1+" +--- +metadata: + name: #@ name + labels: + app.kubernetes.io/name: #@ name +spec: + selector: + matchLabels: + app.kubernetes.io/name: #@ name + template: + metadata: + labels: + app.kubernetes.io/name: #@ name + spec: + volumes: + #@overlay/match by="name" + - name: secret-volume + projected: + sources: + #@overlay/match by=overlay.all, expects="1+" + - secret: + name: #@ data.values.secret diff --git a/scripts/carvel-e2e-azure-psql/cleanup.sh b/scripts/carvel-e2e-azure-psql/cleanup.sh new file mode 100755 index 0000000..2514757 --- /dev/null +++ b/scripts/carvel-e2e-azure-psql/cleanup.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -euo pipefail + +pushd $(dirname $0) + +export PACKAGE_NAMESPACE=${PACKAGE_NAMESPACE:-services} +export APP_NAMESPACE=${APP_NAMESPACE:-services} +VALUES=$(kubectl -n ${PACKAGE_NAMESPACE} get packageinstalls.packaging.carvel.dev ${PACKAGE_METADATA_NAME} -o jsonpath='{.spec.values[0].secretRef.name}') +NAME=$(kubectl -n ${PACKAGE_NAMESPACE} get secrets ${VALUES} -o jsonpath='{.data.values\.yml}' | base64 -d | yq .name) +APP_NAME=${APP_NAME:-${NAME}} +TIMEOUT="15m" +CHECK_INTERVAL="10s" + +kubectl -n ${APP_NAMESPACE} delete deployments.apps ${APP_NAME} || true + +SA=${PACKAGE_METADATA_NAME} +INSTALL_NAME=${PACKAGE_METADATA_NAME} + +echo ">> Uninstall package" +kctrl package installed delete -n ${PACKAGE_NAMESPACE} -i ${INSTALL_NAME} --wait-timeout ${TIMEOUT} --wait-check-interval ${CHECK_INTERVAL} -y + +echo ">> Remove RBAC" +ytt -f ./rbac.ytt.yml -v serviceAccount=${SA} -v namespace=${PACKAGE_NAMESPACE} | kubectl delete -f - + +popd diff --git a/scripts/carvel-e2e-azure-psql/rbac.ytt.yml b/scripts/carvel-e2e-azure-psql/rbac.ytt.yml new file mode 100644 index 0000000..22cd66a --- /dev/null +++ b/scripts/carvel-e2e-azure-psql/rbac.ytt.yml @@ -0,0 +1,53 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:overlay", "overlay") + +#@ namespace = data.values.namespace if "namespace" in data.values else "services" +#@ serviceAccountName = data.values.serviceAccount +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: #@ serviceAccountName +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: #@ serviceAccountName +rules: +- apiGroups: ["dbforpostgresql.azure.com"] + resources: ["flexibleservers","flexibleserversdatabases","flexibleserversfirewallrules"] + verbs: ["*"] +- apiGroups: ["resources.azure.com"] + resources: ["resourcegroups"] + verbs: ["*"] +- apiGroups: ["secretgen.carvel.dev", "secretgen.k14s.io"] + resources: ["secrettemplates","passwords"] + verbs: ["*"] +- apiGroups: [""] + resources: ["serviceaccounts","configmaps"] + verbs: ["*"] +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles","rolebindings"] + verbs: ["*"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: #@ serviceAccountName + "-binding" +subjects: +- kind: ServiceAccount + name: #@ serviceAccountName +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: #@ serviceAccountName + + +#@overlay/match by=overlay.all, expects="1+" +--- +metadata: + #@overlay/match missing_ok=True + namespace: #@ namespace diff --git a/scripts/carvel-e2e-azure-psql/test.sh b/scripts/carvel-e2e-azure-psql/test.sh new file mode 100755 index 0000000..03fee47 --- /dev/null +++ b/scripts/carvel-e2e-azure-psql/test.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -euo pipefail + +pushd $(dirname $0) + +SECRET_NAME=$1 +TEST_APP_NAME=${2:-${SECRET_NAME}} +APP_NAMESPACE=${APP_NAMESPACE:-default} + +echo ">> Installing Test Application" +MANIFEST="https://raw.githubusercontent.com/joostvdg/spring-boot-postgres/main/kubernetes/deployment.yaml" +curl -sSfL ${MANIFEST} | ytt -f - -f app-overlay.ytt.yml -v name=${TEST_APP_NAME} -v secret=${SECRET_NAME} | kubectl apply -n ${APP_NAMESPACE} -f - + +kubectl -n ${APP_NAMESPACE} get deployments.apps + +echo ">> Waiting on Test Application: ${TEST_APP_NAME}" +kubectl -n ${APP_NAMESPACE} get pods +kubectl -n ${APP_NAMESPACE} wait --for=condition=Ready pods -l app.kubernetes.io/name=${TEST_APP_NAME} --timeout=300s + +kubectl -n ${APP_NAMESPACE} describe pods -l app.kubernetes.io/name=${TEST_APP_NAME} +sleep 10 + +echo ">> Starting Port Forward" +kubectl -n ${APP_NAMESPACE} port-forward deployment/${TEST_APP_NAME} 8080 & +PORT_FORWARD_PID=$! +trap "echo '>> Killing Port Forward' && kill -9 ${PORT_FORWARD_PID}" EXIT + +sleep 10 + +echo ">> Testing Application" +curl -sSfL "http://localhost:8080" && echo +echo -n ">> Writing record: " && curl -sSfL --header "Content-Type: application/json" --request POST --data '{"name":"Piet"}' http://localhost:8080/create && echo +echo -n ">> Writing record: " && curl -sSfL --header "Content-Type: application/json" --request POST --data '{"name":"Andrea"}' http://localhost:8080/create && echo + +HTTP_RESULT=$(curl -sSfL "http://localhost:8080") +[ $(jq 'map(select(.name == "Piet")) | length'<<<$HTTP_RESULT) -eq 1 ] +[ $(jq 'map(select(.name == "Andrea")) | length'<<<$HTTP_RESULT) -eq 1 ] + +echo "TEST PASSED" + +popd diff --git a/scripts/crossplane-e2e-mongodb.sh b/scripts/crossplane-e2e-azure-mongodb.sh similarity index 100% rename from scripts/crossplane-e2e-mongodb.sh rename to scripts/crossplane-e2e-azure-mongodb.sh diff --git a/scripts/crossplane-e2e-mongodb/claim-instance.sh b/scripts/crossplane-e2e-azure-mongodb/claim-instance.sh similarity index 100% rename from scripts/crossplane-e2e-mongodb/claim-instance.sh rename to scripts/crossplane-e2e-azure-mongodb/claim-instance.sh diff --git a/scripts/crossplane-e2e-mongodb/cleanup.sh b/scripts/crossplane-e2e-azure-mongodb/cleanup.sh similarity index 100% rename from scripts/crossplane-e2e-mongodb/cleanup.sh rename to scripts/crossplane-e2e-azure-mongodb/cleanup.sh diff --git a/scripts/crossplane-e2e-mongodb/install-package.sh b/scripts/crossplane-e2e-azure-mongodb/install-package.sh similarity index 100% rename from scripts/crossplane-e2e-mongodb/install-package.sh rename to scripts/crossplane-e2e-azure-mongodb/install-package.sh diff --git a/scripts/crossplane-e2e-mongodb/test.sh b/scripts/crossplane-e2e-azure-mongodb/test.sh similarity index 100% rename from scripts/crossplane-e2e-mongodb/test.sh rename to scripts/crossplane-e2e-azure-mongodb/test.sh From bbfe36eed64c20d281124b4e0da6fcce2ec7f2ee Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Mon, 2 Jan 2023 17:30:07 +0100 Subject: [PATCH 21/41] warn about pre-release untested packages --- .github/workflows/publish-packages.yml | 83 ++-------- .github/workflows/reusable-bump-version.yml | 14 +- ...ml => reusable-carvel-publish-package.yml} | 10 +- .../reusable-carvel-publish-repo.yml | 151 ++++++++++++++++++ .../reusable-carvel-repo-publish.yml | 78 --------- .github/workflows/reusable-carvel-test.yml | 43 +++-- .../workflows/reusable-crossplane-publish.yml | 6 - .../workflows/reusable-crossplane-test.yml | 10 +- packages/aws/carvel/elasticache/test | 1 + packages/azure/carvel/psql/test | 1 + 10 files changed, 216 insertions(+), 181 deletions(-) rename .github/workflows/{reusable-carvel-publish.yml => reusable-carvel-publish-package.yml} (94%) create mode 100644 .github/workflows/reusable-carvel-publish-repo.yml delete mode 100644 .github/workflows/reusable-carvel-repo-publish.yml create mode 100644 packages/aws/carvel/elasticache/test create mode 100644 packages/azure/carvel/psql/test diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index 6629c38..0b9afec 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -48,14 +48,14 @@ jobs: package_path: ${{ matrix.path }} run-test: ${{ github.event_name == 'pull_request' }} - carvel-publish: + carvel-publish-package: needs: - bump-version - list-packages if: needs.list-packages.outputs.carvel_publish == 'true' strategy: matrix: ${{ fromJson(needs.list-packages.outputs.carvel) }} - uses: ./.github/workflows/reusable-carvel-publish.yml + uses: ./.github/workflows/reusable-carvel-publish-package.yml with: package_name: ${{ matrix.name }} package_provider: ${{ matrix.provider }} @@ -63,78 +63,17 @@ jobs: packages_basedir: ${{ needs.list-packages.outputs.basedir }} package_path: ${{ matrix.path }} - carvel-prepare-repo: - needs: - - bump-version - - list-packages - - carvel-publish - runs-on: ubuntu-latest - - outputs: - repo-branch-name: ${{ steps.git.outputs.REPO_BRANCH_NAME }} - - env: - PACKAGE_VERSION: ${{ needs.bump-version.outputs.version }} - REPO_BRANCH_NAME: carvel-repo-${{ needs.bump-version.outputs.version }} - - steps: - - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: '0' - - - name: Prepare git client - id: git - run: | - git config --global user.email "${{ github.event.repository.name }}@${{ github.repository_owner }}.github.io" - git config --global user.name "GitHub Actions Workflow" - - sleep 5 && git fetch --all - - echo "CURRENT_BRANCH=$(git branch --show-current)" >> $GITHUB_ENV - echo "REPO_BRANCH_NAME=$REPO_BRANCH_NAME" >> $GITHUB_OUTPUT - - - name: Create branch and pull request - run: | - # create new branch - git checkout -b ${REPO_BRANCH_NAME} - - # include the changes from all the packages' branches - # and remove them at the end of the script - BRANCHES= - for PACKAGE_DIR in $(jq -r '.include[]|.path' <<<$PACKAGES_LIST); do - PACKAGE_METADATA_NAME=$(yq e '.metadata.name' ${PACKAGES_BASEDIR}/${PACKAGE_DIR}/package-metadata.yml) - PACKAGE_BRANCH="${PACKAGE_METADATA_NAME}/${PACKAGE_VERSION}" - BRANCHES="${BRANCHES} ${PACKAGE_BRANCH}" - git rebase origin/${PACKAGE_BRANCH} - done - trap "git push --delete origin ${BRANCHES}" EXIT - - # prepare a proper commit message - COMMIT_MESSAGE="$(git log --pretty=format:'%s' ${CURRENT_BRANCH}..HEAD)" - - # discard the existing commits from packages' branches but do keep the changed files - git reset --soft ${CURRENT_BRANCH} - - # create a new commit on the repo branch and push it to origin - git add . && git commit -m "${COMMIT_MESSAGE}" && git push -u origin ${REPO_BRANCH_NAME} - - # create the pull request against CURRENT_BRANCH - gh pr create --base ${CURRENT_BRANCH} --title "Carvel repository release at commit ${{ github.sha }}" --body "${COMMIT_MESSAGE}" - env: - PACKAGES_BASEDIR: ${{ needs.list-packages.outputs.basedir }} - PACKAGES_LIST: ${{ needs.list-packages.outputs.carvel }} - GH_TOKEN: ${{ github.token }} - - carvel-repo-publish: + carvel-publish-repo: if: needs.list-packages.outputs.carvel_publish == 'true' - uses: ./.github/workflows/reusable-carvel-repo-publish.yml + uses: ./.github/workflows/reusable-carvel-publish-repo.yml needs: + - list-packages - bump-version - - carvel-prepare-repo + - carvel-publish-package with: - git_ref: ${{ needs.carvel-prepare-repo.outputs.repo-branch-name }} + packages_basedir: ${{ needs.list-packages.outputs.basedir }} + packages_list: ${{ needs.list-packages.outputs.carvel }} + package_version: ${{ needs.bump-version.outputs.version }} repo_version: ${{ needs.bump-version.outputs.version }} release: true @@ -144,7 +83,7 @@ jobs: needs: - list-packages - bump-version - - carvel-repo-publish + - carvel-publish-repo strategy: matrix: ${{ fromJson(needs.list-packages.outputs.carvel) }} with: @@ -154,4 +93,6 @@ jobs: package_version: ${{ needs.bump-version.outputs.version }} packages_basedir: ${{ needs.list-packages.outputs.basedir }} package_path: ${{ matrix.path }} + pull_request_number: ${{ needs.carvel-publish-repo.outputs.pull_request_number}} + package_prerelease: ${{ needs.bump-version.outputs.is_prerelease }} secrets: inherit diff --git a/.github/workflows/reusable-bump-version.yml b/.github/workflows/reusable-bump-version.yml index d9b0edb..a5d8759 100644 --- a/.github/workflows/reusable-bump-version.yml +++ b/.github/workflows/reusable-bump-version.yml @@ -13,6 +13,9 @@ on: version: value: ${{ jobs.bump-version.outputs.version }} description: The new published version + is_prerelease: + value: ${{ jobs.bump-version.outputs.is_prerelease }} + description: Whether the version is a pre-release or not jobs: @@ -21,7 +24,8 @@ jobs: runs-on: ubuntu-latest outputs: - version: ${{ steps.bump.outputs.new_tag }} + version: ${{ steps.bump.outputs.new_tag }} + is_prerelease: ${{ steps.pre.outputs.is_prerelease }} steps: @@ -38,3 +42,11 @@ jobs: RELEASE_BRANCHES: main PRERELEASE: true DEFAULT_BUMP: ${{ inputs.default_bump }} + + - name: Check pre-release + id: pre + run: | + [[ "$TAG" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] && is_prerelease=false || is_prerelease=true + echo "is_prerelease=${is_prerelease}" >> $GITHUB_OUTPUT + env: + TAG: ${{ steps.bump.outputs.new_tag }} \ No newline at end of file diff --git a/.github/workflows/reusable-carvel-publish.yml b/.github/workflows/reusable-carvel-publish-package.yml similarity index 94% rename from .github/workflows/reusable-carvel-publish.yml rename to .github/workflows/reusable-carvel-publish-package.yml index 9ae4814..cdcfce8 100644 --- a/.github/workflows/reusable-carvel-publish.yml +++ b/.github/workflows/reusable-carvel-publish-package.yml @@ -25,13 +25,7 @@ on: type: string description: Directory path to hold the contents of the Carvel repository default: repository - kubernetes-version: - type: string - default: v1.24.6 - kind-version: - type: string - default: v0.16.0 - kctrl-version: + kctrl_version: type: string default: v0.43.2 @@ -75,7 +69,7 @@ jobs: uses: vmware-tanzu/carvel-setup-action@v1 with: token: ${{ secrets.GITHUB_TOKEN }} - kctrl: ${{ inputs.kctrl-version }} + kctrl: ${{ inputs.kctrl_version }} - name: Kctrl Release run: make kctrl-release diff --git a/.github/workflows/reusable-carvel-publish-repo.yml b/.github/workflows/reusable-carvel-publish-repo.yml new file mode 100644 index 0000000..bfb13d9 --- /dev/null +++ b/.github/workflows/reusable-carvel-publish-repo.yml @@ -0,0 +1,151 @@ +name: Reusable workflow for publishing Carvel packages repository + +on: + + workflow_call: + + inputs: + packages_basedir: + type: string + description: Packages base directory path + default: packages + packages_list: + type: string + description: List of packages that have been changed + required: true + repo_dir: + type: string + default: repository + release: + type: boolean + default: false + repo_name: + type: string + default: carvel-reference-packages + repo_version: + type: string + required: true + package_version: + type: string + required: true + kctrl_version: + type: string + default: v0.43.2 + + outputs: + registry: + value: ${{ jobs.publish.outputs.registry }} + repository: + value: ${{ jobs.publish.outputs.repository }} + pull_request_number: + value: ${{ jobs.prepare.outputs.pull_request_number }} + +env: + CARVEL_REPO_DIR: ${{ inputs.repo_dir }} + CARVEL_REPO_NAME: ${{ inputs.repo_name }} + CARVEL_REPO_REGISTRY: ghcr.io + CARVEL_REPO_REPOSITORY: ${{ github.repository }}/${{ inputs.repo_name }} + CARVEL_REPO_VERSION: ${{ inputs.repo_version }} + REPO_BRANCH_NAME: carvel-repo-${{ inputs.repo_version }} + PACKAGE_VERSION: ${{ inputs.package_version }} + +jobs: + + prepare: + runs-on: ubuntu-latest + + outputs: + pull_request_number: ${{ steps.pr.outputs.PULL_REQUEST_NUMBER }} + + steps: + + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: '0' + + - name: Prepare git client + id: git + run: | + git config --global user.email "${{ github.event.repository.name }}@${{ github.repository_owner }}.github.io" + git config --global user.name "GitHub Actions Workflow" + + sleep 5 && git fetch --all + + echo "CURRENT_BRANCH=$(git branch --show-current)" >> $GITHUB_ENV + + - name: Create branch and pull request + id: pr + run: | + # create new branch + git checkout -b ${REPO_BRANCH_NAME} + + # include the changes from all the packages' branches + # and remove them at the end of the script + BRANCHES= + for PACKAGE_DIR in $(jq -r '.include[]|.path' <<<$PACKAGES_LIST); do + PACKAGE_METADATA_NAME=$(yq e '.metadata.name' ${PACKAGES_BASEDIR}/${PACKAGE_DIR}/package-metadata.yml) + PACKAGE_BRANCH="${PACKAGE_METADATA_NAME}/${PACKAGE_VERSION}" + BRANCHES="${BRANCHES} ${PACKAGE_BRANCH}" + git rebase origin/${PACKAGE_BRANCH} + done + trap "git push --delete origin ${BRANCHES}" EXIT + + # prepare a proper commit message + COMMIT_MESSAGE="$(git log --pretty=format:'%s' ${CURRENT_BRANCH}..HEAD)" + + # discard the existing commits from packages' branches but do keep the changed files + git reset --soft ${CURRENT_BRANCH} + + # create a new commit on the repo branch and push it to origin + git add . && git commit -m "${COMMIT_MESSAGE}" && git push -u origin ${REPO_BRANCH_NAME} + + # create the pull request against CURRENT_BRANCH + gh pr create --base ${CURRENT_BRANCH} --title "Carvel repository release at commit ${{ github.sha }}" --body "${COMMIT_MESSAGE}" + + # get pull request number + echo "PULL_REQUEST_NUMBER=$(gh pr view --json number -q '.number' ${REPO_BRANCH_NAME})" >> $GITHUB_OUTPUT + env: + PACKAGES_BASEDIR: ${{ inputs.packages_basedir }} + PACKAGES_LIST: ${{ inputs.packages_list }} + GH_TOKEN: ${{ github.token }} + + publish: + runs-on: ubuntu-latest + needs: + - prepare + + outputs: + registry: ${{ env.CARVEL_REPO_REGISTRY }} + repository: ${{ env.CARVEL_REPO_REPOSITORY }} + + steps: + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ env.REPO_BRANCH_NAME }} + + - name: Install Carvel tools + uses: vmware-tanzu/carvel-setup-action@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + kctrl: ${{ inputs.kctrl_version }} + + - name: Kctrl Release + id: kctrl + run: | + make kctrl-repo-release + env: + IMGPKG_REGISTRY_HOSTNAME: ${{ env.CARVEL_REPO_REGISTRY }} + IMGPKG_REGISTRY_USERNAME: ${{ github.actor }} + IMGPKG_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + + - name: Release + if: inputs.release + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ env.CARVEL_REPO_VERSION }} + fail_on_unmatched_files: true + files: | + repository/package-repository.yml diff --git a/.github/workflows/reusable-carvel-repo-publish.yml b/.github/workflows/reusable-carvel-repo-publish.yml deleted file mode 100644 index 108d8ad..0000000 --- a/.github/workflows/reusable-carvel-repo-publish.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Reusable workflow for publishing Carvel packages repository - -on: - - workflow_call: - - inputs: - repo_dir: - type: string - default: repository - release: - type: boolean - default: false - repo_name: - type: string - default: carvel-reference-packages - repo_version: - type: string - git_ref: - type: string - default: ${{ github.ref }} - description: Git ref to checkout (defaults to github.ref) - kctrl-version: - type: string - default: v0.43.2 - - outputs: - registry: - value: ${{ jobs.publish.outputs.registry }} - repository: - value: ${{ jobs.publish.outputs.repository }} - -env: - CARVEL_REPO_DIR: ${{ inputs.repo_dir }} - CARVEL_REPO_NAME: ${{ inputs.repo_name }} - CARVEL_REPO_REGISTRY: ghcr.io - CARVEL_REPO_REPOSITORY: ${{ github.repository }}/${{ inputs.repo_name }} - CARVEL_REPO_VERSION: ${{ inputs.repo_version }} - -jobs: - - publish: - runs-on: ubuntu-latest - - outputs: - registry: ${{ env.CARVEL_REPO_REGISTRY }} - repository: ${{ env.CARVEL_REPO_REPOSITORY }} - - steps: - - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ inputs.git_ref }} - - - name: Install Carvel tools - uses: vmware-tanzu/carvel-setup-action@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - kctrl: ${{ inputs.kctrl-version }} - - - name: Kctrl Release - id: kctrl - run: | - make kctrl-repo-release - env: - IMGPKG_REGISTRY_HOSTNAME: ${{ env.CARVEL_REPO_REGISTRY }} - IMGPKG_REGISTRY_USERNAME: ${{ github.actor }} - IMGPKG_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - - - name: Release - if: inputs.release - uses: softprops/action-gh-release@v1 - with: - tag_name: ${{ env.CARVEL_REPO_VERSION }} - fail_on_unmatched_files: true - files: | - repository/package-repository.yml diff --git a/.github/workflows/reusable-carvel-test.yml b/.github/workflows/reusable-carvel-test.yml index b0e7070..7fc15b0 100644 --- a/.github/workflows/reusable-carvel-test.yml +++ b/.github/workflows/reusable-carvel-test.yml @@ -11,6 +11,9 @@ on: package_version: type: string required: true + package_prerelease: + type: string + required: true package_name: type: string required: true @@ -24,16 +27,19 @@ on: package_path: type: string required: true - kubernetes-version: + pull_request_number: + type: string + required: true + kubernetes_version: type: string default: v1.24.6 - kind-version: + kind_version: type: string default: v0.16.0 - kapp-controller-version: + kapp_controller_version: type: string default: v0.43.2 - secretgen-controller-version: + secretgen_controller_version: type: string default: v0.12.0 @@ -60,9 +66,9 @@ jobs: uses: helm/kind-action@v1.4.0 with: verbosity: 5 - version: ${{ inputs.kind-version }} - kubectl_version: ${{ inputs.kubernetes-version }} - node_image: kindest/node:${{ inputs.kubernetes-version }} + version: ${{ inputs.kind_version }} + kubectl_version: ${{ inputs.kubernetes_version }} + node_image: kindest/node:${{ inputs.kubernetes_version }} - name: Verify Cluster run: | @@ -73,8 +79,8 @@ jobs: - name: Install kapp-controller run: | - MANIFEST="https://github.com/vmware-tanzu/carvel-kapp-controller/releases/download/${{ inputs.kapp-controller-version }}/release.yml" - if [[ "${{ inputs.kapp-controller-version }}" == "latest" ]]; then + MANIFEST="https://github.com/vmware-tanzu/carvel-kapp-controller/releases/download/${{ inputs.kapp_controller_version }}/release.yml" + if [[ "${{ inputs.kapp_controller_version }}" == "latest" ]]; then MANIFEST="https://github.com/vmware-tanzu/carvel-kapp-controller/releases/latest/download/release.yml" fi @@ -87,8 +93,8 @@ jobs: - name: Install secretgen-controller run: | - MANIFEST="https://github.com/vmware-tanzu/carvel-secretgen-controller/releases/download/${{ inputs.secretgen-controller-version }}/release.yml" - if [[ "${{ inputs.secretgen-controller-version }}" == "latest" ]]; then + MANIFEST="https://github.com/vmware-tanzu/carvel-secretgen-controller/releases/download/${{ inputs.secretgen_controller_version }}/release.yml" + if [[ "${{ inputs.secretgen_controller_version }}" == "latest" ]]; then MANIFEST="https://github.com/vmware-tanzu/carvel-secretgen-controller/releases/latest/download/release.yml" fi @@ -135,12 +141,25 @@ jobs: run: | export PACKAGE_METADATA_NAME=$(yq e '.metadata.name' ${PACKAGE_DIR}/package-metadata.yml) echo PACKAGE_METADATA_NAME=${PACKAGE_METADATA_NAME} >> $GITHUB_ENV - if [ -x ${SCRIPT} ]; then + + if [[ "$PRERELEASE" =~ ^[tT][rR][uU][eE]$ ]]; then + if [ -x ${SCRIPT} ]; then + ${SCRIPT} + else + # no pre-release test script provided + gh label create -c ffaa00 --force untested + gh pr edit $PULL_REQUEST_NUMBER --add-label untested + gh pr comment $PULL_REQUEST_NUMBER --body "WARNING - package ${PACKAGE_METADATA_NAME}/${PACKAGE_VERSION} has not been tested" + fi + else ${SCRIPT} fi env: SCRIPT: ./scripts/carvel-e2e-${{ inputs.package_provider }}-${{ inputs.package_name }}.sh PACKAGE_DIR: ${{ inputs.packages_basedir }}/${{ inputs.package_path }} + PRERELEASE: ${{ inputs.package_prerelease }} + PULL_REQUEST_NUMBER: ${{ inputs.pull_request_number }} + GH_TOKEN: ${{ github.token }} - name: Cleanup Carvel package if: always() diff --git a/.github/workflows/reusable-crossplane-publish.yml b/.github/workflows/reusable-crossplane-publish.yml index 405180a..704ab5e 100644 --- a/.github/workflows/reusable-crossplane-publish.yml +++ b/.github/workflows/reusable-crossplane-publish.yml @@ -24,12 +24,6 @@ on: run-test: type: boolean default: false - kubernetes-version: - type: string - default: v1.24.6 - kind-version: - type: string - default: v0.16.0 outputs: package_repository: diff --git a/.github/workflows/reusable-crossplane-test.yml b/.github/workflows/reusable-crossplane-test.yml index 798df92..2a81a7f 100644 --- a/.github/workflows/reusable-crossplane-test.yml +++ b/.github/workflows/reusable-crossplane-test.yml @@ -21,10 +21,10 @@ on: package_repository: type: string required: true - kubernetes-version: + kubernetes_version: type: string default: v1.24.6 - kind-version: + kind_version: type: string default: v0.16.0 @@ -52,9 +52,9 @@ jobs: uses: helm/kind-action@v1.4.0 with: verbosity: 5 - version: ${{ inputs.kind-version }} - kubectl_version: ${{ inputs.kubernetes-version }} - node_image: kindest/node:${{ inputs.kubernetes-version }} + version: ${{ inputs.kind_version }} + kubectl_version: ${{ inputs.kubernetes_version }} + node_image: kindest/node:${{ inputs.kubernetes_version }} - name: Verify Cluster run: | diff --git a/packages/aws/carvel/elasticache/test b/packages/aws/carvel/elasticache/test new file mode 100644 index 0000000..9c558e3 --- /dev/null +++ b/packages/aws/carvel/elasticache/test @@ -0,0 +1 @@ +. diff --git a/packages/azure/carvel/psql/test b/packages/azure/carvel/psql/test new file mode 100644 index 0000000..9c558e3 --- /dev/null +++ b/packages/azure/carvel/psql/test @@ -0,0 +1 @@ +. From 7b8b6a32e568d8e476a17f53b789d35148ebdb93 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Tue, 3 Jan 2023 19:00:20 +0100 Subject: [PATCH 22/41] hide secret values from pipeline environment --- .github/workflows/reusable-carvel-test.yml | 28 ++++++++++------------ packages/aws/carvel/elasticache/test | 1 + packages/azure/carvel/psql/test | 1 + 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/reusable-carvel-test.yml b/.github/workflows/reusable-carvel-test.yml index 7fc15b0..a443c2a 100644 --- a/.github/workflows/reusable-carvel-test.yml +++ b/.github/workflows/reusable-carvel-test.yml @@ -121,24 +121,20 @@ jobs: with: secret: ${{ secrets.AZURE_CONFIG }} - - name: Set Azure secret - if: inputs.package_provider == 'azure' - env: - AZURE_CONFIG: ${{ secrets.AZURE_CONFIG }} - run: | - # transforms the keys in the input JSON secret from camelCase to UPPER_SNAKE_CASE - # with the AZURE_ prefix - # in order not to divert from the official ASO docs and the client JSON file generated by az CLI - for k in $(jq -r 'keys[]' <<<"$AZURE_CONFIG"); do - K=$(echo azure_$k | sed -r 's/([a-z0-9])([A-Z])/\1_\L\2/g' | tr '[:lower:]' '[:upper:]') - echo $K=$(jq -r '.'$k <<<"$AZURE_CONFIG") >> $GITHUB_ENV - done - - name: Checkout uses: actions/checkout@v3 - name: Test Carvel package run: | + if [[ "${{ inputs.package_provider }}" == "azure" ]]; then + # transforms the keys in the input JSON secret from camelCase to UPPER_SNAKE_CASE with the AZURE_ prefix + # in order not to divert from the official ASO docs and the client JSON file generated by az CLI + for k in $(jq -r 'keys[]' <<<"$AZURE_CONFIG"); do + K=$(echo azure_$k | sed -r 's/([a-z0-9])([A-Z])/\1_\L\2/g' | tr '[:lower:]' '[:upper:]') + eval export $K=$(jq -r '.'$k <<<"$AZURE_CONFIG") + done + fi + export PACKAGE_METADATA_NAME=$(yq e '.metadata.name' ${PACKAGE_DIR}/package-metadata.yml) echo PACKAGE_METADATA_NAME=${PACKAGE_METADATA_NAME} >> $GITHUB_ENV @@ -147,14 +143,16 @@ jobs: ${SCRIPT} else # no pre-release test script provided - gh label create -c ffaa00 --force untested - gh pr edit $PULL_REQUEST_NUMBER --add-label untested + LABEL=untested + gh label create -c ffaa00 --force $LABEL + gh pr edit $PULL_REQUEST_NUMBER --add-label $LABEL gh pr comment $PULL_REQUEST_NUMBER --body "WARNING - package ${PACKAGE_METADATA_NAME}/${PACKAGE_VERSION} has not been tested" fi else ${SCRIPT} fi env: + AZURE_CONFIG: ${{ secrets.AZURE_CONFIG }} SCRIPT: ./scripts/carvel-e2e-${{ inputs.package_provider }}-${{ inputs.package_name }}.sh PACKAGE_DIR: ${{ inputs.packages_basedir }}/${{ inputs.package_path }} PRERELEASE: ${{ inputs.package_prerelease }} diff --git a/packages/aws/carvel/elasticache/test b/packages/aws/carvel/elasticache/test index 9c558e3..ac860a3 100644 --- a/packages/aws/carvel/elasticache/test +++ b/packages/aws/carvel/elasticache/test @@ -1 +1,2 @@ . +. diff --git a/packages/azure/carvel/psql/test b/packages/azure/carvel/psql/test index 9c558e3..ac860a3 100644 --- a/packages/azure/carvel/psql/test +++ b/packages/azure/carvel/psql/test @@ -1 +1,2 @@ . +. From ae7bb6385700f2ba5db985ed223516690cbc9209 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Wed, 4 Jan 2023 09:52:18 +0100 Subject: [PATCH 23/41] fix azure test script --- .github/workflows/reusable-bump-version.yml | 2 +- config/carvel/package-install/rbac.ytt.yml | 52 ------------------- config/carvel/package-install/schema.ytt.yml | 9 ---- .../psql/config/01-flexible-server.ytt.yml | 2 - .../carvel/psql/config/99-kapp-config.yml | 6 ++- packages/azure/carvel/psql/test | 2 - scripts/carvel-e2e-azure-psql.sh | 11 ++-- 7 files changed, 13 insertions(+), 71 deletions(-) delete mode 100644 config/carvel/package-install/rbac.ytt.yml delete mode 100644 config/carvel/package-install/schema.ytt.yml delete mode 100644 packages/azure/carvel/psql/test diff --git a/.github/workflows/reusable-bump-version.yml b/.github/workflows/reusable-bump-version.yml index a5d8759..c7529dc 100644 --- a/.github/workflows/reusable-bump-version.yml +++ b/.github/workflows/reusable-bump-version.yml @@ -34,7 +34,7 @@ jobs: - name: Bump version and push tag id: bump - uses: anothrNick/github-tag-action@v1 + uses: anothrNick/github-tag-action@1.57.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} WITH_V: false diff --git a/config/carvel/package-install/rbac.ytt.yml b/config/carvel/package-install/rbac.ytt.yml deleted file mode 100644 index e7a6dc6..0000000 --- a/config/carvel/package-install/rbac.ytt.yml +++ /dev/null @@ -1,52 +0,0 @@ -#@ load("@ytt:data", "data") -#@ load("@ytt:overlay", "overlay") - -#@ serviceAccountName = data.values.refName + "-install" ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: #@ serviceAccountName ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: #@ serviceAccountName -rules: -- apiGroups: ["dbforpostgresql.azure.com"] - resources: ["flexibleservers","flexibleserversdatabases","flexibleserversfirewallrules"] - verbs: ["*"] -- apiGroups: ["resources.azure.com"] - resources: ["resourcegroups"] - verbs: ["*"] -- apiGroups: ["secretgen.carvel.dev", "secretgen.k14s.io"] - resources: ["secrettemplates","passwords"] - verbs: ["*"] -- apiGroups: [""] - resources: ["serviceaccounts","configmaps"] - verbs: ["*"] -- apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] -- apiGroups: ["rbac.authorization.k8s.io"] - resources: ["roles","rolebindings"] - verbs: ["*"] ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: #@ serviceAccountName + "-binding" -subjects: -- kind: ServiceAccount - name: #@ serviceAccountName -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: #@ serviceAccountName - - -#@overlay/match by=overlay.all, expects="1+" ---- -metadata: - #@overlay/match missing_ok=True - namespace: #@ data.values.namespace diff --git a/config/carvel/package-install/schema.ytt.yml b/config/carvel/package-install/schema.ytt.yml deleted file mode 100644 index 936d365..0000000 --- a/config/carvel/package-install/schema.ytt.yml +++ /dev/null @@ -1,9 +0,0 @@ -#@data/values-schema ---- -#@schema/desc "The package full name as from .spec.refName" -#@schema/validation min_len=1 -refName: "" - -#@schema/desc "The namespace where the package must be installed" -#@schema/validation min_len=1 -namespace: services diff --git a/packages/azure/carvel/psql/config/01-flexible-server.ytt.yml b/packages/azure/carvel/psql/config/01-flexible-server.ytt.yml index cb0445e..488f218 100644 --- a/packages/azure/carvel/psql/config/01-flexible-server.ytt.yml +++ b/packages/azure/carvel/psql/config/01-flexible-server.ytt.yml @@ -80,8 +80,6 @@ metadata: serviceoperator.azure.com/operator-namespace: #@ data.values.aso_controller_namespace #@ if/end data.values.resource_group.use_existing: serviceoperator.azure.com/reconcile-policy: skip - labels: - kapp.k14s.io/noop: "" spec: azureName: #@ data.values.resource_group.name location: #@ data.values.location diff --git a/packages/azure/carvel/psql/config/99-kapp-config.yml b/packages/azure/carvel/psql/config/99-kapp-config.yml index 95bd6db..7fff562 100644 --- a/packages/azure/carvel/psql/config/99-kapp-config.yml +++ b/packages/azure/carvel/psql/config/99-kapp-config.yml @@ -50,4 +50,8 @@ rebaseRules: - [metadata, annotations, serviceoperator.azure.com/poller-resume-token] type: copy sources: [new, existing] - resourceMatchers: *flexibleServerResources \ No newline at end of file + resourceMatchers: *flexibleServerResources + +ownershipLabelRules: +- path: [metadata, labels] + resourceMatchers: *flexibleServerResources diff --git a/packages/azure/carvel/psql/test b/packages/azure/carvel/psql/test deleted file mode 100644 index ac860a3..0000000 --- a/packages/azure/carvel/psql/test +++ /dev/null @@ -1,2 +0,0 @@ -. -. diff --git a/scripts/carvel-e2e-azure-psql.sh b/scripts/carvel-e2e-azure-psql.sh index c156c5f..be23d7b 100755 --- a/scripts/carvel-e2e-azure-psql.sh +++ b/scripts/carvel-e2e-azure-psql.sh @@ -4,10 +4,13 @@ set -euo pipefail TIMEOUT=${TIMEOUT:-5m} +SCRIPT_FOLDER=$(basename $0 .sh) + export NAME="${NAME:-$(dd if=/dev/urandom bs=20 count=1 2>/dev/null | sha1sum | head -c 20)}" PACKAGE_NAMESPACE=${PACKAGE_NAMESPACE:-services} export APP_NAMESPACE=${APP_NAMESPACE:-services} export APP_NAME=${APP_NAME:-${NAME}} +ASO_CONTROLLER_NAMESPACE="azureserviceoperator-system" pushd $(dirname $0) @@ -24,7 +27,7 @@ cat <$VALUES name: ${NAME} namespace: ${PACKAGE_NAMESPACE} location: ${LOCATION} -aso_controller_namespace: azureserviceoperator-system +aso_controller_namespace: ${ASO_CONTROLLER_NAMESPACE} create_namespace: false server: @@ -52,7 +55,7 @@ SA=${PACKAGE_METADATA_NAME} INSTALL_NAME=${PACKAGE_METADATA_NAME} echo ">> Prepare RBAC" -ytt -f ./carvel-e2e-azure-psql/rbac.ytt.yml -v serviceAccount=${SA} -v namespace=${PACKAGE_NAMESPACE} | kubectl apply -f - +ytt -f ./${SCRIPT_FOLDER}/rbac.ytt.yml -v serviceAccount=${SA} -v namespace=${PACKAGE_NAMESPACE} | kubectl apply -f - echo ">> Install package" kctrl package install -n ${PACKAGE_NAMESPACE} -i ${INSTALL_NAME} -p ${PACKAGE_METADATA_NAME} --version ${PACKAGE_VERSION} --values-file ${VALUES} --service-account-name ${SA} --wait=false @@ -67,12 +70,12 @@ while [ $RESTARTS_COUNT -lt $RESTARTS_MAX ]; do else # ASO needs to be kicked because it conflicts with kapp-controller for taking ownership of Azure resources let RESTARTS_COUNT=$RESTARTS_COUNT+1 - kubectl -n azureserviceoperator-system rollout restart deployments.apps azureserviceoperator-controller-manager + kubectl -n ${ASO_CONTROLLER_NAMESPACE} rollout restart deployments.apps azureserviceoperator-controller-manager fi done # run test SECRET_NAME="${NAME}-bindable" -./carvel-e2e-azure-psql/test.sh ${SECRET_NAME} ${APP_NAME} +./${SCRIPT_FOLDER}/test.sh ${SECRET_NAME} ${APP_NAME} popd From e9dd3f3d2d9b186e61c19c934b58a85e4f32f444 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Thu, 5 Jan 2023 16:36:55 +0100 Subject: [PATCH 24/41] fixup! fix azure test script --- packages/aws/carvel/elasticache/test | 1 + .../psql/config/01-flexible-server.ytt.yml | 2 +- scripts/carvel-e2e-azure-psql.sh | 65 +++++++++++++------ scripts/carvel-e2e-azure-psql/cleanup.sh | 3 + 4 files changed, 49 insertions(+), 22 deletions(-) diff --git a/packages/aws/carvel/elasticache/test b/packages/aws/carvel/elasticache/test index ac860a3..bd812ad 100644 --- a/packages/aws/carvel/elasticache/test +++ b/packages/aws/carvel/elasticache/test @@ -1,2 +1,3 @@ . . +. diff --git a/packages/azure/carvel/psql/config/01-flexible-server.ytt.yml b/packages/azure/carvel/psql/config/01-flexible-server.ytt.yml index 488f218..2e8263b 100644 --- a/packages/azure/carvel/psql/config/01-flexible-server.ytt.yml +++ b/packages/azure/carvel/psql/config/01-flexible-server.ytt.yml @@ -233,7 +233,7 @@ spec: app.kubernetes.io/component: #@ data.values.name app.kubernetes.io/instance: "$(.server.metadata.name)" services.apps.tanzu.vmware.com/class: azure-postgres - type: postgresql + type: servicebinding.io/postgresql stringData: type: postgresql port: "5432" diff --git a/scripts/carvel-e2e-azure-psql.sh b/scripts/carvel-e2e-azure-psql.sh index be23d7b..45e5b43 100755 --- a/scripts/carvel-e2e-azure-psql.sh +++ b/scripts/carvel-e2e-azure-psql.sh @@ -2,24 +2,44 @@ set -euo pipefail -TIMEOUT=${TIMEOUT:-5m} +INSTALL_TIMEOUT="20m" SCRIPT_FOLDER=$(basename $0 .sh) export NAME="${NAME:-$(dd if=/dev/urandom bs=20 count=1 2>/dev/null | sha1sum | head -c 20)}" -PACKAGE_NAMESPACE=${PACKAGE_NAMESPACE:-services} +export PACKAGE_NAMESPACE=${PACKAGE_NAMESPACE:-services} export APP_NAMESPACE=${APP_NAMESPACE:-services} -export APP_NAME=${APP_NAME:-${NAME}} -ASO_CONTROLLER_NAMESPACE="azureserviceoperator-system" +APP_NAME=${APP_NAME:-${NAME}} +export ASO_CONTROLLER_NAMESPACE="azureserviceoperator-system" + +[ -z "${PACKAGE_METADATA_NAME:-}" ] && echo "Environment variable PACKAGE_METADATA_NAME is not defined" && exit 1 +[ -z "${PACKAGE_VERSION:-}" ] && echo "Environment variable PACKAGE_VERSION is not defined" && exit 1 + +while true; do + case "${1:-}" in + --skip-aso-install) + SKIP_ASO_INSTALL=1 ; shift ;; + *) + break ;; + esac +done pushd $(dirname $0) -# install ASO and dependencies -./carvel-azure-install-aso.sh +if [ -z ${SKIP_ASO_INSTALL:-} ]; then + # install ASO and dependencies + ./carvel-azure-install-aso.sh +fi + +echo ">> Prepare package values" -# install package LOCATION="${LOCATION:-westeurope}" -PUBLIC_IP="$(curl -sSf https://api.ipify.org)" + +JOBNAME="get-public-ip-${NAME}" +kubectl create job ${JOBNAME} --image curlimages/curl -- curl -sSf https://api.ipify.org +kubectl wait --for=condition=Complete=True job ${JOBNAME} +PUBLIC_IP="$(kubectl logs jobs/${JOBNAME})" +kubectl delete job ${JOBNAME} VALUES=$(mktemp) cat <$VALUES @@ -52,7 +72,7 @@ trap "rm ${VALUES}" EXIT kubectl create namespace ${PACKAGE_NAMESPACE} || true SA=${PACKAGE_METADATA_NAME} -INSTALL_NAME=${PACKAGE_METADATA_NAME} +export INSTALL_NAME=${PACKAGE_METADATA_NAME} echo ">> Prepare RBAC" ytt -f ./${SCRIPT_FOLDER}/rbac.ytt.yml -v serviceAccount=${SA} -v namespace=${PACKAGE_NAMESPACE} | kubectl apply -f - @@ -60,19 +80,22 @@ ytt -f ./${SCRIPT_FOLDER}/rbac.ytt.yml -v serviceAccount=${SA} -v namespace=${PA echo ">> Install package" kctrl package install -n ${PACKAGE_NAMESPACE} -i ${INSTALL_NAME} -p ${PACKAGE_METADATA_NAME} --version ${PACKAGE_VERSION} --values-file ${VALUES} --service-account-name ${SA} --wait=false -RESTARTS_MAX=4 -RESTARTS_COUNT=0 -while [ $RESTARTS_COUNT -lt $RESTARTS_MAX ]; do - echo ">> Waiting for stack ${NAME} to reconcile..." - kubectl -n ${PACKAGE_NAMESPACE} wait --for=condition=ReconcileSucceeded --timeout=${TIMEOUT} packageinstalls.packaging.carvel.dev ${INSTALL_NAME} && AGAIN=0 || AGAIN=1 - if [ $AGAIN -eq 0 ]; then - RESTARTS_COUNT=$RESTARTS_MAX - else - # ASO needs to be kicked because it conflicts with kapp-controller for taking ownership of Azure resources - let RESTARTS_COUNT=$RESTARTS_COUNT+1 - kubectl -n ${ASO_CONTROLLER_NAMESPACE} rollout restart deployments.apps azureserviceoperator-controller-manager - fi +timeout --foreground -s TERM $INSTALL_TIMEOUT bash -c ' +INIT_TIMEOUT_SECONDS=${TIMEOUT:-60} +MAX_TIMEOUT_SECONDS=${TIMEOUT:-300} +TIMEOUT_SECONDS=${INIT_TIMEOUT_SECONDS} +while true; do + echo ">> Waiting for stack ${NAME} to reconcile for ${TIMEOUT_SECONDS} seconds..." + kubectl -n ${PACKAGE_NAMESPACE} wait --for=condition=ReconcileSucceeded --timeout="${TIMEOUT_SECONDS}s" packageinstalls.packaging.carvel.dev ${INSTALL_NAME} && break || true + + # the cloud controller needs to be kicked because it might conflict with kapp-controller for taking ownership of cloud resources + kubectl -n ${ASO_CONTROLLER_NAMESPACE} rollout restart deployments.apps azureserviceoperator-controller-manager + + let TEMP_TIMEOUT=${TIMEOUT_SECONDS}*2 + [[ ${TEMP_TIMEOUT} > ${MAX_TIMEOUT_SECONDS} ]] && TEMP_TIMEOUT=${MAX_TIMEOUT_SECONDS} + TIMEOUT_SECONDS=${TEMP_TIMEOUT} done +' # run test SECRET_NAME="${NAME}-bindable" diff --git a/scripts/carvel-e2e-azure-psql/cleanup.sh b/scripts/carvel-e2e-azure-psql/cleanup.sh index 2514757..03f632f 100755 --- a/scripts/carvel-e2e-azure-psql/cleanup.sh +++ b/scripts/carvel-e2e-azure-psql/cleanup.sh @@ -4,6 +4,8 @@ set -euo pipefail pushd $(dirname $0) +[ -z "${PACKAGE_METADATA_NAME:-}" ] && echo "Environment variable PACKAGE_METADATA_NAME is not defined" && exit 1 + export PACKAGE_NAMESPACE=${PACKAGE_NAMESPACE:-services} export APP_NAMESPACE=${APP_NAMESPACE:-services} VALUES=$(kubectl -n ${PACKAGE_NAMESPACE} get packageinstalls.packaging.carvel.dev ${PACKAGE_METADATA_NAME} -o jsonpath='{.spec.values[0].secretRef.name}') @@ -12,6 +14,7 @@ APP_NAME=${APP_NAME:-${NAME}} TIMEOUT="15m" CHECK_INTERVAL="10s" + kubectl -n ${APP_NAMESPACE} delete deployments.apps ${APP_NAME} || true SA=${PACKAGE_METADATA_NAME} From 31a1124280362980c2cc0611ac07269c0116ca9b Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Tue, 10 Jan 2023 19:06:57 +0100 Subject: [PATCH 25/41] create repo pr only for release branch --- .github/workflows/publish-packages.yml | 1 + .../reusable-carvel-publish-repo.yml | 18 +++- .gitignore | 99 +++++++++++++++++++ Makefile | 75 -------------- packages/README.md | 26 ++--- .../aws/carvel/elasticache/build-values.yml | 7 -- packages/aws/carvel/elasticache/test | 2 - packages/azure/carvel/psql/test | 1 + scripts/carvel-aws-install-ack.sh | 51 ++++++++++ scripts/carvel-e2e-aws-elasticache.sh.skip | 99 +++++++++++++++++++ .../cleanup.sh | 44 +++++++++ .../rbac.ytt.yml | 50 ++++++++++ 12 files changed, 365 insertions(+), 108 deletions(-) delete mode 100644 packages/aws/carvel/elasticache/build-values.yml create mode 100644 packages/azure/carvel/psql/test create mode 100755 scripts/carvel-aws-install-ack.sh create mode 100755 scripts/carvel-e2e-aws-elasticache.sh.skip create mode 100755 scripts/carvel-e2e-aws-elasticache.skip/cleanup.sh create mode 100644 scripts/carvel-e2e-aws-elasticache.skip/rbac.ytt.yml diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index 0b9afec..a7e7352 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -75,6 +75,7 @@ jobs: packages_list: ${{ needs.list-packages.outputs.carvel }} package_version: ${{ needs.bump-version.outputs.version }} repo_version: ${{ needs.bump-version.outputs.version }} + prepare_repo_pr: ${{ needs.bump-version.outputs.is_prerelease == 'false' }} release: true carvel-test: diff --git a/.github/workflows/reusable-carvel-publish-repo.yml b/.github/workflows/reusable-carvel-publish-repo.yml index bfb13d9..1d950d6 100644 --- a/.github/workflows/reusable-carvel-publish-repo.yml +++ b/.github/workflows/reusable-carvel-publish-repo.yml @@ -25,6 +25,9 @@ on: repo_version: type: string required: true + prepare_repo_pr: + type: boolean + default: false package_version: type: string required: true @@ -74,8 +77,8 @@ jobs: echo "CURRENT_BRANCH=$(git branch --show-current)" >> $GITHUB_ENV - - name: Create branch and pull request - id: pr + - name: Create branch + id: branch run: | # create new branch git checkout -b ${REPO_BRANCH_NAME} @@ -99,6 +102,15 @@ jobs: # create a new commit on the repo branch and push it to origin git add . && git commit -m "${COMMIT_MESSAGE}" && git push -u origin ${REPO_BRANCH_NAME} + env: + PACKAGES_BASEDIR: ${{ inputs.packages_basedir }} + PACKAGES_LIST: ${{ inputs.packages_list }} + + - name: Create pull request + id: pr + if: inputs.prepare_repo_pr + run: | + COMMIT_MESSAGE="$(git log --pretty=format:'%s' HEAD~..HEAD)" # create the pull request against CURRENT_BRANCH gh pr create --base ${CURRENT_BRANCH} --title "Carvel repository release at commit ${{ github.sha }}" --body "${COMMIT_MESSAGE}" @@ -106,8 +118,6 @@ jobs: # get pull request number echo "PULL_REQUEST_NUMBER=$(gh pr view --json number -q '.number' ${REPO_BRANCH_NAME})" >> $GITHUB_OUTPUT env: - PACKAGES_BASEDIR: ${{ inputs.packages_basedir }} - PACKAGES_LIST: ${{ inputs.packages_list }} GH_TOKEN: ${{ github.token }} publish: diff --git a/.gitignore b/.gitignore index 8b63b0a..de1f7ab 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,102 @@ **/.src/ **/.package/ pkgrepo-build.yml + +# Created by https://www.toptal.com/developers/gitignore/api/macos,linux,windows,visualstudiocode +# Edit at https://www.toptal.com/developers/gitignore?templates=macos,linux,windows,visualstudiocode + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.toptal.com/developers/gitignore/api/macos,linux,windows,visualstudiocode + diff --git a/Makefile b/Makefile index 349e13f..bfb3d10 100644 --- a/Makefile +++ b/Makefile @@ -1,78 +1,3 @@ -# LOCAL_VERSION = $(shell git describe --tags --always) -# PACKAGE_VERSION ?= "0.0.0-${LOCAL_VERSION}" -# PACKAGE_REPOSITORY ?= "vmware-tanzu-labs/trp-azure-psql" -# PACKAGE_REGISTRY ?= "ghcr.io" -# CLOUD_NAME = azure -# -# PACKAGES_DIR ?= ./packages -# PACKAGE_DIR ?= ${PACKAGES_DIR}/${PACKAGE_NAME} -# RELEASE_DIR = ${PACKAGE_DIR}/.release -# -# PACKAGE_BUILD_BASE_DIR = ./package-build -# PACKAGE_BUILD_DIR = ${PACKAGE_BUILD_BASE_DIR}/${PACKAGE_NAME} -# -# PACKAGE_TEST_DATA_DIR_NAME = test-data -# REPOSITORY_DIR = ../tap-reference-packages-repository -# REPOSITORY_CLOUD_DIR = ${REPOSITORY_DIR}/repository/packages/${CLOUD_NAME} -# -# SHELL := $(shell which bash) -# -# KDEV_SA_NAME=development -# KDEV_SA_NAMESPACE=default -# -# DEV_PACKAGE_NAME ?= ${PACKAGE_NAME} -# DEV_NAMESPACE ?= default - - - -# clean: -# rm -f ${PACKAGE_DIR}/package-kdev.yml || true -# rm -f ${PACKAGE_DIR}/package-build.yml || true -# rm -f ${PACKAGE_DIR}/package-resources.yml || true -# rm -rf ${PACKAGE_BUILD_DIR} || true -# rm -rf ${PACKAGE_DIR}/carvel-artifacts/ || true -# rm -rf ${PACKAGE_DIR}/bundle-* || true -# rm -rf ${PACKAGES_REPO_STAGING_DIR} || true -# mkdir -p ${PACKAGES_REPO_STAGING_DIR} -# -# dev: clean -# kubectl create namespace ${DEV_PACKAGE_NAME} || true -# ytt -f ${PACKAGE_DIR}/config -f ${PACKAGE_DIR}/test-data | kbld -f - | kapp deploy -a ${DEV_PACKAGE_NAME} -n ${DEV_NAMESPACE} --debug -y -f - -# -# dev-cleanup: -# kapp delete -a ${DEV_PACKAGE_NAME} -n ${DEV_NAMESPACE} -y -# -# kdev-prepare: clean -# yq e '... comments="" | .spec.template.spec.template[] |= select(.ytt != null).ytt.paths += "test-data"' ${PACKAGE_DIR}/package.yml > ${PACKAGE_DIR}/package-kdev.yml -# cd ${PACKAGE_DIR} && ytt -f package-kdev.yml -f package-metadata.yml -f package-install.yml > package-resources.yml -# -# kdev: kdev-prepare -# cd ${PACKAGE_DIR} && kctrl dev -f package-resources.yml -l -# -# kdev-cleanup: kdev-prepare -# cd ${PACKAGE_DIR} && kctrl dev -f package-resources.yml -l --delete -# -# create-kdev-sa: -# kubectl create serviceaccount ${KDEV_SA_NAME} -n ${KDEV_SA_NAMESPACE} || true -# kubectl delete clusterrolebinding kdev-cluster-admin || true -# kubectl create clusterrolebinding kdev-cluster-admin --clusterrole=cluster-admin --serviceaccount=${KDEV_SA_NAMESPACE}:${KDEV_SA_NAME} -# -# repo-prepare: -# mkdir -p ${REPOSITORY_CLOUD_DIR} -# -# repo-package-copy: repo-prepare -# cp -a ${PACKAGES_REPO_STAGING_DIR}/packages/$(shell yq e .metadata.name ${PACKAGE_DIR}/package-metadata.yml)/ ${REPOSITORY_CLOUD_DIR}/${PACKAGE_NAME} - - - -# imgpkg-push-prepare: -# mkdir -p ${PACKAGE_BUILD_DIR}/.imgpkg -# cp -a ${PACKAGE_DIR}/config ${PACKAGE_BUILD_DIR} -# kbld -f ${PACKAGE_BUILD_DIR}/config/ --imgpkg-lock-output ${PACKAGE_BUILD_DIR}/.imgpkg/images.yml -# -# imgpkg-push: imgpkg-push-prepare -# imgpkg push -b ${PACKAGE_REGISTRY}/${PACKAGE_REPOSITORY}:${PACKAGE_VERSION} -f ${PACKAGE_BUILD_DIR}/ - PACKAGES_BASEDIR ?= packages PACKAGE_DIR ?= ${PACKAGES_BASEDIR}/${PACKAGE_PROVIDER}/${PACKAGE_PACKAGING}/${PACKAGE_NAME} diff --git a/packages/README.md b/packages/README.md index 7ef20f4..8e2e961 100644 --- a/packages/README.md +++ b/packages/README.md @@ -26,7 +26,6 @@ packages ├── aws │ └── carvel │ └── elasticache -│ ├── build-values.yml │ ├── config │ │ ├── 00-schema.yml │ │ ├── 01-replication-group.ytt.yml @@ -54,7 +53,7 @@ The contents of the `//` directory depends on the spe ### Carvel -The `kctrl` utility from [Carvel suite](https://carvel.dev) is being used to author packages, which makes use of other tools in the suite as well (i.e. `ytt`). +The `kctrl` utility from [Carvel suite](https://carvel.dev) is being used to author packages in combination with other tools in the suite (i.e. `ytt`). The metadata files required to create the package are stored as [ytt templates](../config/carvel/) to easily define a standard to create multiple packages. @@ -67,28 +66,19 @@ that is required to fill some fields in other template files, in a DRY fashion. *DRY: Don't Repeat Yourself -The `config` directory inside the package home holds the ytt files that define the resources that will be created by the package installation -as well as the schema definition for the input values. +The `config` directory inside the package home holds the ytt files that define the resources +that will be created by the package installation as well as the schema definition for the input values. **N.B. The name of such a folder MUST be `config`, as it is hardcoded in the mentioned templates.** -If some sort of validation of the input data has been implemented in the ytt files, you also need -a sample values file to feed `kctrl` with at building time, otherwise it won't succeed. -Those sample data are just used at this stage and won't be included in the package itself. -The sample values file name, if required, MUST be stored into the `PACKAGE_BUILD_VALUES` variable. - -**N.B. By default, the CI workflow assumes that such is called `build-values.yml`, therefore packages -undergoing continuous integration MUST use this name.** - -The command used to build the package and publish it to the container registry is `make kctrl-release`, which uses a few environment variables -for configuration. +The command used to build the package and publish it to the container registry is `make kctrl-release`, +which uses a few environment variables for configuration. The following snippet defines a quick way of building all the `carvel` packages in the `packages` folder. Additionally, it uploads the packages to the ghcr.io service in the `org/repository` repository (do adjust it to your own account), in a hierarchy that reflects the filesystem. ```sh PACKAGE_REGISTRY="ghcr.io" -PACKAGE_BUILD_VALUES="build-values.yml" for PACKAGE_DIR in $(find packages -type d -mindepth 3 -maxdepth 3); do if [[ $(cut -d/ -f3 <<<${PACKAGE_DIR}) == "carvel" ]]; then PACKAGE_REPOSITORY="org/repository/${PACKAGE_DIR#packages/}" @@ -97,8 +87,7 @@ for PACKAGE_DIR in $(find packages -type d -mindepth 3 -maxdepth 3); do done ``` -The prerequisites to that are that you must be authenticated to ghcr.io and you must have the correct permissions -to write packages to `org`. +You must be authenticated to ghcr.io and you must have the correct permissions to write packages to `org`. The authentication credentials can be provided either via [Docker login][docker-login] or via [`imgpkg` environment variables][imgpkg-auth-env], for example: ```sh @@ -109,6 +98,3 @@ export IMGPKG_REGISTRY_PASSWORD="my-personal-access-token" [imgpkg-auth-env]: https://carvel.dev/imgpkg/docs/v0.34.0/auth/#via-environment-variables [docker-login]: https://docs.docker.com/engine/reference/commandline/login/ - -### Crossplane - diff --git a/packages/aws/carvel/elasticache/build-values.yml b/packages/aws/carvel/elasticache/build-values.yml deleted file mode 100644 index 64dc4d5..0000000 --- a/packages/aws/carvel/elasticache/build-values.yml +++ /dev/null @@ -1,7 +0,0 @@ -#@data/values ---- -name: elasticache -namespace: default -cacheSubnetGroupName: default -vpcSecurityGroupIDs: - - vpc-0123456789 diff --git a/packages/aws/carvel/elasticache/test b/packages/aws/carvel/elasticache/test index bd812ad..9c558e3 100644 --- a/packages/aws/carvel/elasticache/test +++ b/packages/aws/carvel/elasticache/test @@ -1,3 +1 @@ . -. -. diff --git a/packages/azure/carvel/psql/test b/packages/azure/carvel/psql/test new file mode 100644 index 0000000..9c558e3 --- /dev/null +++ b/packages/azure/carvel/psql/test @@ -0,0 +1 @@ +. diff --git a/scripts/carvel-aws-install-ack.sh b/scripts/carvel-aws-install-ack.sh new file mode 100755 index 0000000..d30b164 --- /dev/null +++ b/scripts/carvel-aws-install-ack.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -euo pipefail + +[ -z "$SERVICE" ] && ( echo "The SERVICE environment variable must be defined" ; exit 1 ) + +CREDENTIALS=${CREDENTIALS:-$HOME/.aws/credentials} + +helm_install() { + echo ">> Install the AWS $SERVICE Controller for Kubernetes" + + aws ecr-public get-login-password --region us-east-1 | \ + helm registry login --username AWS --password-stdin public.ecr.aws + + helm upgrade --install \ + ack-$SERVICE-controller \ + oci://public.ecr.aws/aws-controllers-k8s/$SERVICE-chart \ + --create-namespace \ + --namespace $ACK_NAMESPACE \ + --version=$RELEASE_VERSION \ + --set=aws.region=$AWS_REGION \ + $* +} + +SERVICE=$(tr '[:upper:]' '[:lower:]' <<<$SERVICE) +RELEASE_VERSION=`curl -sL https://api.github.com/repos/aws-controllers-k8s/$SERVICE-controller/releases/latest | jq -r .tag_name` +ACK_NAMESPACE=${ACK_NAMESPACE:-ack-system} +AWS_REGION=${AWS_REGION:-eu-central-1} + +echo ">> Define ACK authentication" +# https://aws-controllers-k8s.github.io/community/docs/user-docs/authentication/ + +if [ ! -z "${AWS_ACCESS_KEY_ID:-}" -a ! -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then + echo "WARNING - Using AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_SESSION_TOKEN environment variables" + helm_install + + kubectl -n ${ACK_NAMESPACE} set env deployments.apps -l app.kubernetes.io/instance=ack-${SERVICE}-controller \ + AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ + AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ + AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" + +elif [ -r "${CREDENTIALS}" ]; then + echo "Using shared credentials file" + SECRET="aws-credentials" + kubectl create namespace $ACK_NAMESPACE || true + kubectl -n ${ACK_NAMESPACE} delete secret ${SECRET} &>/dev/null || true + kubectl -n ${ACK_NAMESPACE} create secret generic ${SECRET} --from-file credentials=${CREDENTIALS} + + helm_install --set=aws.credentials.secretName=${SECRET} + +fi diff --git a/scripts/carvel-e2e-aws-elasticache.sh.skip b/scripts/carvel-e2e-aws-elasticache.sh.skip new file mode 100755 index 0000000..2972d44 --- /dev/null +++ b/scripts/carvel-e2e-aws-elasticache.sh.skip @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +set -euo pipefail + +INSTALL_TIMEOUT="20m" + +SCRIPT_FOLDER=$(basename $0 .sh) + +export NAME="${NAME:-$(dd if=/dev/urandom bs=20 count=1 2>/dev/null | sha1sum | head -c 20)}" +export PACKAGE_NAMESPACE=${PACKAGE_NAMESPACE:-services} +export APP_NAMESPACE=${APP_NAMESPACE:-services} +export APP_NAME=${APP_NAME:-${NAME}} +export ACK_NAMESPACE="ack-system" + +[ -z "${PACKAGE_METADATA_NAME:-}" ] && echo "Environment variable PACKAGE_METADATA_NAME is not defined" && exit 1 +[ -z "${PACKAGE_VERSION:-}" ] && echo "Environment variable PACKAGE_VERSION is not defined" && exit 1 +[ -z "${CACHE_SUBNET_GROUP_NAME:-}" ] && echo "Environment variable CACHE_SUBNET_GROUP_NAME is not defined" && exit 1 + +while true; do + case "${1:-}" in + --skip-ack-install) + SKIP_ACK_INSTALL=1 ; shift ;; + *) + break ;; + esac +done + +pushd $(dirname $0) + +if [ -z ${SKIP_ACK_INSTALL:-} ]; then + # install ACK + ./carvel-aws-install-ack.sh +fi + +echo ">> Prepare security group" + +JOBNAME="get-public-ip-${NAME}" +kubectl create job ${JOBNAME} --image curlimages/curl -- curl -sSf https://api.ipify.org +kubectl wait --for=condition=Complete=True job ${JOBNAME} +PUBLIC_IP="$(kubectl logs jobs/${JOBNAME})" +kubectl delete job ${JOBNAME} + +VPC_ID=$(aws elasticache describe-cache-subnet-groups --cache-subnet-group-name $CACHE_SUBNET_GROUP_NAME --query 'CacheSubnetGroups[0].VpcId' --output text) +SG_NAME="elasticache-test-${NAME}" + +# make sure the security group does not exist before creating a new one +{ + aws ec2 describe-security-groups --filters 'Name="vpc-id",Values="'${VPC_ID}'"' 'Name="group-name",Values="'${SG_NAME}'"' --query 'SecurityGroups[0].GroupId' --output text | xargs aws ec2 delete-security-group --group-id +} || true +SG_ID=$(aws ec2 create-security-group --group-name "${SG_NAME}" --description "elasticache security group for testing" --vpc-id $VPC_ID --output text --query GroupId) +# trap "aws ec2 delete-security-group --group-id $SG_ID" EXIT +aws ec2 authorize-security-group-ingress --group-id $SG_ID --cidr ${PUBLIC_IP}/32 --protocol tcp --port 6379 + + +VALUES=$(mktemp) +cat <$VALUES +--- +name: test-${NAME} +namespace: ${PACKAGE_NAMESPACE} +cacheSubnetGroupName: ${CACHE_SUBNET_GROUP_NAME} +cacheNodeType: cache.t2.micro +vpcSecurityGroupIDs: + - ${SG_ID} +EOF + +# install package +kubectl create namespace ${PACKAGE_NAMESPACE} || true + +SA=${PACKAGE_METADATA_NAME} +export INSTALL_NAME=${PACKAGE_METADATA_NAME} + +echo ">> Prepare RBAC" +ytt -f ./${SCRIPT_FOLDER}/rbac.ytt.yml -v serviceAccount=${SA} -v namespace=${PACKAGE_NAMESPACE} | kubectl apply -f - + +echo ">> Install package" +kctrl package install -n ${PACKAGE_NAMESPACE} -i ${INSTALL_NAME} -p ${PACKAGE_METADATA_NAME} --version ${PACKAGE_VERSION} --values-file ${VALUES} --service-account-name ${SA} --wait=false + +timeout --foreground -s TERM $INSTALL_TIMEOUT bash -c ' +INIT_TIMEOUT_SECONDS=${TIMEOUT:-60} +MAX_TIMEOUT_SECONDS=${TIMEOUT:-300} +TIMEOUT_SECONDS=${INIT_TIMEOUT_SECONDS} +while true; do + echo ">> Waiting for stack ${NAME} to reconcile for ${TIMEOUT_SECONDS} seconds..." + kubectl -n ${PACKAGE_NAMESPACE} wait --for=condition=ReconcileSucceeded --timeout="${TIMEOUT_SECONDS}s" packageinstalls.packaging.carvel.dev ${INSTALL_NAME} && break || true + + # the cloud controller needs to be kicked because it might conflict with kapp-controller for taking ownership of cloud resources + kubectl -n ${ACK_NAMESPACE} rollout restart deployments.apps -l app.kubernetes.io/instance=ack-${SERVICE}-controller + + let TEMP_TIMEOUT=${TIMEOUT_SECONDS}*2 + [[ ${TEMP_TIMEOUT} > ${MAX_TIMEOUT_SECONDS} ]] && TEMP_TIMEOUT=${MAX_TIMEOUT_SECONDS} + TIMEOUT_SECONDS=${TEMP_TIMEOUT} +done +' + +# run test +SECRET_NAME="${NAME}-bindable" +# ./${SCRIPT_FOLDER}/test.sh ${SECRET_NAME} ${APP_NAME} + +popd diff --git a/scripts/carvel-e2e-aws-elasticache.skip/cleanup.sh b/scripts/carvel-e2e-aws-elasticache.skip/cleanup.sh new file mode 100755 index 0000000..0829981 --- /dev/null +++ b/scripts/carvel-e2e-aws-elasticache.skip/cleanup.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -euo pipefail + +pushd $(dirname $0) + +[ -z "${PACKAGE_METADATA_NAME:-}" ] && echo "Environment variable PACKAGE_METADATA_NAME is not defined" && exit 1 + +export PACKAGE_NAMESPACE=${PACKAGE_NAMESPACE:-services} +export APP_NAMESPACE=${APP_NAMESPACE:-services} + +INSTALL_NAME=${PACKAGE_METADATA_NAME} + +VALUES_SECRET_NAME=$(kubectl -n ${PACKAGE_NAMESPACE} get packageinstalls.packaging.carvel.dev ${INSTALL_NAME} -o jsonpath='{.spec.values[0].secretRef.name}') +PACKAGE_DATA=$(kubectl -n ${PACKAGE_NAMESPACE} get secrets ${VALUES_SECRET_NAME} -o jsonpath='{.data}') +PACKAGE_VALUES=$(jq -e '.data."values.yml"' <<<$PACKAGE_DATA || jq -e '.data."values.yaml"' <<<$PACKAGE_DATA) + +NAME=$(base64 -d <<<$PACKAGE_VALUES | yq .name) +APP_NAMESPACE=$(base64 -d <<<$PACKAGE_VALUES | yq .namespace) + +APP_NAME=${APP_NAME:-${NAME}} +TIMEOUT="5m" +CHECK_INTERVAL="10s" + + +kubectl -n ${APP_NAMESPACE} delete deployments.apps ${APP_NAME} || true + +SA=${PACKAGE_METADATA_NAME} + +echo ">> Uninstall package" +kctrl package installed delete -n ${PACKAGE_NAMESPACE} -i ${INSTALL_NAME} --wait-timeout ${TIMEOUT} --wait-check-interval ${CHECK_INTERVAL} -y || { + # the default user gets locked into the ACK.Terminal state because it can be deleted BEFORE its group is deleted + # and when a resource get to that state there's no way to get out of it + # in order to delete the kubernetes resource I need to remove the finalizers + kubectl -n ${APP_NAMESPACE} patch users.elasticache.services.k8s.aws ${NAME}-default --type=json --patch '[{"op":"remove","path":"/metadata/finalizers"}]' + DEFAULT_USER_ID=$(kubectl -n ${APP_NAMESPACE} get users.elasticache.services.k8s.aws ${NAME}-default -o jsonpath='{.spec.userID}') + aws elasticache delete-user --user-id $DEFAULT_USER_ID + kubectl -n ${APP_NAMESPACE} delete users.elasticache.services.k8s.aws ${NAME}-default +} + +echo ">> Remove RBAC" +ytt -f ./rbac.ytt.yml -v serviceAccount=${SA} -v namespace=${PACKAGE_NAMESPACE} | kubectl delete -f - + +popd diff --git a/scripts/carvel-e2e-aws-elasticache.skip/rbac.ytt.yml b/scripts/carvel-e2e-aws-elasticache.skip/rbac.ytt.yml new file mode 100644 index 0000000..67c2ae8 --- /dev/null +++ b/scripts/carvel-e2e-aws-elasticache.skip/rbac.ytt.yml @@ -0,0 +1,50 @@ +#@ load("@ytt:data", "data") +#@ load("@ytt:overlay", "overlay") + +#@ namespace = data.values.namespace if "namespace" in data.values else "services" +#@ serviceAccountName = data.values.serviceAccount +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: #@ serviceAccountName +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: #@ serviceAccountName +rules: +- apiGroups: ["elasticache.services.k8s.aws"] + resources: ["*"] + verbs: ["*"] +- apiGroups: ["secretgen.carvel.dev", "secretgen.k14s.io"] + resources: ["secrettemplates","passwords"] + verbs: ["*"] +- apiGroups: [""] + resources: ["serviceaccounts","configmaps"] + verbs: ["*"] +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] +- apiGroups: ["rbac.authorization.k8s.io"] + resources: ["roles","rolebindings"] + verbs: ["*"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: #@ serviceAccountName + "-binding" +subjects: +- kind: ServiceAccount + name: #@ serviceAccountName +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: #@ serviceAccountName + + +#@overlay/match by=overlay.all, expects="1+" +--- +metadata: + #@overlay/match missing_ok=True + namespace: #@ namespace From fbcd7e4c4f511ded8c21ea6fb53373b2c4cfe9fb Mon Sep 17 00:00:00 2001 From: Joost van der Griendt Date: Tue, 10 Jan 2023 17:45:04 +0100 Subject: [PATCH 26/41] document multicloud psql crossplane package --- docs/crossplane/providers/helm.md | 75 ++ docs/crossplane/providers/kubernetes.md | 73 ++ docs/crossplane/providers/terraform.md | 56 + .../multicloud/packages/psql/create.md | 681 ++++++++++++ .../multicloud/packages/psql/index.md | 969 ++++++++++++++++++ mkdocs.yml | 8 + 6 files changed, 1862 insertions(+) create mode 100644 docs/crossplane/providers/helm.md create mode 100644 docs/crossplane/providers/kubernetes.md create mode 100644 docs/crossplane/providers/terraform.md create mode 100644 docs/usecases/multicloud/packages/psql/create.md create mode 100644 docs/usecases/multicloud/packages/psql/index.md diff --git a/docs/crossplane/providers/helm.md b/docs/crossplane/providers/helm.md new file mode 100644 index 0000000..ffab1f2 --- /dev/null +++ b/docs/crossplane/providers/helm.md @@ -0,0 +1,75 @@ +--- +title: Helm provider +description: How to install and configure Crossplane Helm provider +Hero: Install Crossplane Helm provider +--- + +The [Helm provider](https://marketplace.upbound.io/providers/crossplane-contrib/provider-helm/latest) is a community provider. +As the name suggests, it lets you manage Helm chart installations with Crossplane. + +## Install + +You can install the provider via the [up](https://docs.upbound.io/cli/) CLI or a Kubernetes manifest. + +=== "Upbound CLI" + ```sh + up controlplane provider install \ + xpkg.upbound.io/crossplane-contrib/provider-helm:v0.12.0 + ``` + +=== "Kubernetes Manifest" + ```sh + cat <=v0.12.0" + + crossplane: + #@schema/title "CrossplaneNamespace" + #@schema/desc "The namespace where crossplane controller is installed" + namespace: upbound-system + version: '^v1.10' + + #@schema/title "StoreConfig" + #@schema/desc "Details of the StoreConfig" + storeConfig: + #@schema/title "StoreConfig Name" + #@schema/desc "The name of the StoreConfig" + name: "default" + ``` + +### Solving For Re-usable Snippets + +In the templating section, you can see how we reuse values from the schema to avoid duplication and misconfiguration. + +Even so, the number of shared data structures between the **Compositions** is significant. +The solution for the **ConnetionDetails**, for example, is something each **Composition** requires. + +So there is a need for even more reuse. +Not just the data values, but entire data structures, for example, the Composition ***Resources***. + +For that purpose, we use the **YTT** features [Load](https://carvel.dev/ytt/docs/v0.44.0/lang-ref-load/) and [Functions](https://carvel.dev/ytt/docs/v0.44.0/lang-ref-def/). +It lets you create functions in separate files you import into other YTT files. + +!!! Info "YTT Library" + We could also have opted to use a **YTT** [Library](https://carvel.dev/ytt/docs/v0.44.0/lang-ref-ytt-library/). + + Which does practically the same thing but requires more steps to use. + So we opted for the solution below. + +To use this, we do the following: + +1. Create a file named `.lib.yml` +1. Add ***Functions*** in this file that generate the Crossplane Composition Resource snippets +1. Load the file and the desired functions in the YTT _data_ file (a) file starting with `#@ load("@ytt:data", "data")`) +1. Use the functions as if they are defined in this YTT data file + +Let's look at some examples to clarify what we did. +First, it's a function in a _library_ module. + +!!! Example "shared.lib.yml" + + For clarity, we removed some lines and highlighted the noteworthy lines. + + We **start** a function with `#@ def ()`. + + We **end** a function with `#@ end`. + + We can use any defined input parameters directly by name. + We determine the value by the order in which the caller supplies them. + + In this case, we have one parameter, `infra`. + This lets us create a dynamic label on the secret based on the type of infrastructure the **Composition** implements (e.g., `azure`, `aws`, `kubernetes`). + + ```yaml hl_lines="1 15 18" title="shared.lib.yml" + #@ def labelsForSecret(infra): + name: connectionSecret + base: + apiVersion: kubernetes.crossplane.io/v1alpha1 + kind: Object + spec: + forProvider: + manifest: + apiVersion: v1 + kind: Secret + spec: {} + metadata: + labels: + services.apps.tanzu.vmware.com/class: multicloud-psql + services.apps.tanzu.vmware.com/infra: #@ infra + patches: + ... + #@ end + ``` + +We use the library as follows: + +1. We import it using the `#@ load()` feature. Using the relative path of the file and the functions we want to use. + ```yaml + #@ load("shared.lib.yml", "labelsForSecret") + ``` +1. Then, we call the function where we want its _output_ to be. + ```yaml + resources: + - #@ labelsForSecret("aws") + ``` + +!!! Important + By default, a function returns the data structure that it defines. + + If you want to return a single value, you can do so via the `return` statement like this: + + ```yaml + #@ def TLSSecretName(domain): + #@ return str(domain).replace(".", "-") + "-tls" + #@ end + ``` + +The functions can use everything **YTT** has to offer. + +This makes it an excellent place to handle specific logic for **Crossplane Compositions**. +For example, in the case of the AWS RDS instance, we have private and public variants. + +This means that while most values for the RDS instance definition are the same, some change if it needs to be public. + +!!! Example "AWS Composition Public/Private Switch" + + When we make the RDS instance publicly available, we need to set `spec.forProvider.publiclyAccessible` to true. + + We also need to tell **Crossplan** which ***SubnetGroup*** it has to use. + When we make the RDS instance public, we create the ***SubnetGroup*** ourselves and let **Crossplane** manage the reference: + + ```yaml + #@ if/end publiclyAccessible: + dbSubnetGroupNameSelector: + matchControllerRef: true + ``` + _ps. `#@ if/end` means it does an if/else/end expression for a single line only_ + + If private, we expect you to supply the name of an existing one. + We can use the `not` keyword to reverse the `if/end`, so we end up with two variations of the same snippet. + + ```yaml + #@ if/end not publiclyAccessible: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.aws.dbSubnetGroupName + toFieldPath: spec.forProvider.dbSubnetGroupName + ``` + + ```yaml hl_lines="13 19 25" title="aws-composition.lib.yml" + #@ def rdsInstance(crossplaneNamespace, providerConfigRef, publiclyAccessible): + name: rdsinstance + base: + apiVersion: rds.aws.upbound.io/v1beta1 + kind: Instance + spec: + forProvider: + engine: postgres + instanceClass: db.t3.micro + passwordSecretRef: + key: password + namespace: #@ crossplaneNamespace + publiclyAccessible: #@ publiclyAccessible + skipFinalSnapshot: true + storageEncrypted: false + allocatedStorage: 10 + vpcSecurityGroupIdSelector: + matchControllerRef: true + #@ if/end publiclyAccessible: + dbSubnetGroupNameSelector: + matchControllerRef: true + providerConfigRef: + name: #@ providerConfigRef + patches: + #@ if/end not publiclyAccessible: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.aws.dbSubnetGroupName + toFieldPath: spec.forProvider.dbSubnetGroupName + ... + #@ end + ``` + +Let's look at the two AWS examples (private and public) to see how this works out for the **Composition** files. + +#### AWS Private Example + +```yaml title="aws-composition-private.ytt.yml" +#@ load("@ytt:data", "data") +--- +#@ load("aws-composition.lib.yml", "rdsInstance", "securityGroup", "securityGroupRule") +#@ load("shared.lib.yml", "labelsForSecret", "tfProviderConfig", "tfWorkspace") + +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: #@ data.values.providers.aws.name + "-" + data.values.cloudServiceBindingType + "-private" + labels: + crossplane.io/xrd: #@ data.values.xrd.names.plural + "." + data.values.xrd.group + provider: #@ data.values.providers.aws.name + database: #@ data.values.cloudServiceBindingType + connectivity: "private" +spec: + writeConnectionSecretsToNamespace: #@ data.values.crossplane.namespace + compositeTypeRef: + apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version + kind: #@ data.values.xrd.names.kind + resources: + - #@ labelsForSecret("aws") + - #@ tfProviderConfig(data.values.crossplane.namespace) + - #@ tfWorkspace(data.values.crossplane.namespace) + - #@ rdsInstance(data.values.crossplane.namespace, data.values.providers.aws.configRef, False) + - #@ securityGroup() + - #@ securityGroupRule() +``` + +#### AWS Public Example + +!!! Info "Convenience Variables" + + YTT supports the use of convenience variables. + + Below you can see we make ample use of them to make the subnet parameters clearer to understand and separate. + +```yaml title="aws-composition-public.ytt.yml" +#@ load("@ytt:data", "data") +--- +#@ load("aws-composition.lib.yml", "rdsInstance", "securityGroup", "securityGroupRule", "subnet", "routeTableAssociation", "routeTable", "route", "subnetGroup") +#@ load("shared.lib.yml", "labelsForSecret", "tfProviderConfig", "tfWorkspace") + +#@ subnetASuffix = '-a' +#@ subnetAFormat = 'subnet-a-%s' +#@ availabilityZoneAFormat = '%sa' +#@ cidrBlockFieldA = 'spec.parameters.aws.public.subnetACidrBlock' + +#@ subnetBSuffix = '-b' +#@ subnetBFormat = 'subnet-b-%s' +#@ availabilityZoneBFormat = '%sb' +#@ cidrBlockFieldB = 'spec.parameters.aws.public.subnetBCidrBlock' + +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: #@ data.values.providers.aws.name + "-" + data.values.cloudServiceBindingType + "-public" + labels: + crossplane.io/xrd: #@ data.values.xrd.names.plural + "." + data.values.xrd.group + provider: #@ data.values.providers.aws.name + database: #@ data.values.cloudServiceBindingType + connectivity: "public" +spec: + writeConnectionSecretsToNamespace: #@ data.values.crossplane.namespace + compositeTypeRef: + apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version + kind: #@ data.values.xrd.names.kind + resources: + - #@ labelsForSecret("aws") + - #@ tfProviderConfig(data.values.crossplane.namespace) + - #@ tfWorkspace(data.values.crossplane.namespace) + - #@ rdsInstance(data.values.crossplane.namespace, data.values.providers.aws.configRef, True) + - #@ securityGroup() + - #@ securityGroupRule() + - #@ routeTable() + - #@ subnetGroup() + - #@ subnet(subnetASuffix, subnetAFormat, availabilityZoneAFormat, cidrBlockFieldA) + - #@ routeTableAssociation(subnetASuffix, subnetAFormat) + - #@ subnet(subnetBSuffix, subnetBFormat, availabilityZoneBFormat, cidrBlockFieldB) + - #@ routeTableAssociation(subnetBSuffix, subnetBFormat) + - #@ route('route-%s') +``` diff --git a/docs/usecases/multicloud/packages/psql/index.md b/docs/usecases/multicloud/packages/psql/index.md new file mode 100644 index 0000000..deafafa --- /dev/null +++ b/docs/usecases/multicloud/packages/psql/index.md @@ -0,0 +1,969 @@ +--- +title: Consuming Postgresql Multicloud +--- + +## Why Multicloud + +There are various reasons why companies are running applications in multiple clouds. +We count data centers as local or private clouds. + +It is increasingly common for companies to run Kubernetes clusters in multiple locations - some clusters in the data center and others in AWS, for example. + +When (service) consumers do not care about the specific performance parameters of a database, you can simplify onboarding and relocation pain by providing a generic API. + +This package is an example of such a generic API. +It holds as long as the consumers want _a database_ and there is no need (yet) to fine-tune it. + +Good examples are PoCs, preview environments, or applications where the performance bottleneck lies elsewhere. + +## Pre-requisites + +This package supports three platforms; Kubernetes, Azure, and AWS. +Therefore, we split the prerequisites into shared requirements and cloud-specific (Azure and AWS, respectively). + +### Shared + +!!! Warning "Terraform provider configuration included" + The package creates a **ProviderConfig** for the Terraform provider for you. To ensure it is unique, it has the UID from Composite Resource (`XPostgreSQLInstance`) as the name. + + So when installing the **Terraform Provider** (to meet the prerequisites), there is no need to create a **ProviderConfig**. + +* Kubernetes 1.23+ +* [Crossplane 1.10+](/tanzu-application-platform-reference-service-packages/crossplane/) +* [Crossplane Kubernetes provider (Community)](/tanzu-application-platform-reference-service-packages/crossplane/providers/kubernetes/) +* [Crossplane Terraform provider (Upbound official)](/tanzu-application-platform-reference-service-packages/crossplane/providers/terraform/) + +### Kubernetes + +* [Crossplane Helm provider (Community)](/tanzu-application-platform-reference-service-packages/crossplane/providers/helm/) + +### Azure + +When wanting to consume the Azure FlexibleServer variant, you also need the following: + +* Azure credentials +* [Crossplane Azure provider (Upbound official)](/tanzu-application-platform-reference-service-packages/crossplane/providers/azure/) + +### AWS + +* AWS credentials +* [Crossplane AWS provider (Upbound official)](/tanzu-application-platform-reference-service-packages/crossplane/providers/aws/) + +## Install Crossplane Package + +We start by setting some environment variables and installing our Crossplane package (Configuration CR). + +```sh +CONFIG_NAME="trp-multicloud-psql" +CONFIG_IMAGE="ghcr.io/vmware-tanzu-labs/trp-multicloud-psql" +CONFIG_VERSION="0.1.0-rc-4" +CROSSPLANE_NAMESPACE="upbound-system" +``` + +```sh +cat < resource from Crossplane). + + ```sh + kubectl describe xpostgresqlinstances.multi.ref.services.apps.tanzu.vmware.com/${CLAIM_NAME} + ``` + +!!! Failure "Cannot render composed resource" + If you get a warning like this in the **Composite Resource**: + + ```sh + cannot render composed resource from resource template at index 1: cannot use dry-run create to name composed resource: workspaces.tf.crossplane.io is forbidden: User "system:serviceaccount:upbound-system:crossplane" cannot create resource "workspaces" in API group "tf.crossplane.io" at the cluster scope + ``` + + Sometimes the service account of Crossplane or a Crossplane Provider does not have enough permissions to create the resources. + You can either fix this quickly and dirty (for PoCs) like below: + + ```sh + kubectl create clusterrolebinding crossplane-admin-binding --clusterrole cluster-admin --serviceaccount="upbound-system:crossplane" || true + ``` + + Or find the Service Account for the specific provider (they have generated names) and update its RBAC configuration concisely. + + ```sh + SA=$(kubectl -n upbound-system get sa -o name | grep provider-terraform | sed -e 's|serviceaccount\/|upbound-system:|g') + ``` + +!!! Success + If the wait commands returns with `Conditions met`, the package is ready. + + You can now continue with [test without TAP](#test-withouth-tap) or [test with TAP and STK](#test-with-tap-stk). + +## Test Without TAP + +Tanzu Application Platform (TAP) and the Services Toolkit (STK) are great tools for usage at scale. +It can be rather cumbersome to test a single service package. + +So below is an example of a Kubernetes deployment that directly consumes the secret created by Crossplane. +It uses the [Service Bindings](https://servicebinding.io/) specification (Spring library) to translate the secret to the parameters needed by the database API. + +### Install the Test application + +```sh hl_lines="21 22 23 39 40 41 42 49" +cat < workload.yml + ``` + + ```sh + kubectl apply -f workload.yml + ``` + +To see the logs: +```sh +tanzu apps workload tail spring-boot-postgres-01 +``` + +To get the status: + +```sh +tanzu apps workload get spring-boot-postgres-01 +``` + +Tap creates the deployment when the build and config writer workflows are complete. + +To see their pods: + +```sh +kubectl get pod -l app.kubernetes.io/part-of=spring-boot-postgres-01 +``` + +Then you can wait for the application to be ready via the `kubectl` CLI. + +```sh +kubectl wait --for=condition=ready \ + pod -l app.kubernetes.io/component=run,app.kubernetes.io/part-of=spring-boot-postgres-01 \ + --timeout=180s \ + --namespace ${TAP_DEV_NAMESPACE} +``` + +Ensure there is only one deployment active: + +```sh +kubectl get pod --namespace ${TAP_DEV_NAMESPACE} \ + -l app.kubernetes.io/component=run,app.kubernetes.io/part-of=spring-boot-postgres-01 +``` + +Which should list a single deployment with Ready 2/2: + +```sh +NAME READY STATUS RESTARTS AGE +spring-boot-postgres-01-00002-deployment-8cd56bdc8-gb44n 2/2 Running 0 6m11s +``` + +We then collect the name of the **Pod** or copy it from the command-line output. + +```sh +POD_NAME=$(kubectl get pod \ + --namespace ${TAP_DEV_NAMESPACE} \ + -l app.kubernetes.io/component=run,app.kubernetes.io/part-of=spring-boot-postgres-01 \ + -o jsonpath="{.items[0].metadata.name}") +``` + +### Test Application + +Use the `kubectl` CLI to create a port forward. + +```sh +kubectl port-forward pod/${POD_NAME} --namespace ${TAP_DEV_NAMESPACE} 8080 +``` + +And then open another terminal to test the application: + +```sh +curl -s "http://localhost:8080" +``` + +Which should return an empty list `[]`. +Add a value that gets stored in the MongoDB instance. + +```sh +curl --header "Content-Type: application/json" \ + --request POST --data '{"name":"Alice"}' \ + http://localhost:8080/create +``` + +Making another GET request should return the stored entry: + +```sh +curl -s "http://localhost:8080" +``` + +!!! Success + We have gone through all the steps. You can now clean up all the resources. + +## Cleanup + +```sh +CLAIM_NAME="postgresql-0001" +CONFIG_NAME="trp-multicloud-psql" +``` + +!!! Danger "Delete TAP Workload" + ```sh + tanzu apps workload delete spring-boot-postgres-01 || true + ``` + + +!!! Danger "Delete STK Claim" + ```sh + tanzu service claim delete multi-psql-claim-01 || true + kubectl delete ClusterInstanceClass multi-psql multi-psql-kubernetes multi-psql-kubernetes-12 || true + ``` + + +!!! Danger "Delete Crossplane Claim & Package" + ```sh + kubectl delete postgresqlinstances ${CLAIM_NAME} || true + ``` + + ```sh + kubectl delete configuration ${CONFIG_NAME} || true + ``` + +!!! Danger "Delete Crossplane resources" + + ```sh + kubectl delete providerconfigs.helm.crossplane.io default || true + kubectl delete providerconfigs.kubernetes.crossplane.io default || true + + kubectl delete providers.pkg.crossplane.io crossplane-contrib-provider-helm || true + kubectl delete providers.pkg.crossplane.io crossplane-contrib-provider-kubernetes || true + kubectl delete providers.pkg.crossplane.io crossplane-contrib-provider-terraform || true + ``` + +This should not be required, but in case the resources for Azure are not cleaned up when deleting the package: + +!!! Danger "Delete Azure Resources" + ```sh + kubectl delete flexibleserverconfigurations.dbforpostgresql.azure.upbound.io -l crossplane.io/claim-name=${CLAIM_NAME} --force --grace-period=0 || true + kubectl delete flexibleserverdatabases.dbforpostgresql.azure.upbound.io -l crossplane.io/claim-name=${CLAIM_NAME} --force --grace-period=0 || true + kubectl delete flexibleserverfirewallrules.dbforpostgresql.azure.upbound.io -l crossplane.io/claim-name=${CLAIM_NAME} --force --grace-period=0 || true + + FLEXIBLE_SERVER_NAME=$(kubectl get flexibleserver.dbforpostgresql.azure.upbound.io -l crossplane.io/claim-name=${CLAIM_NAME} -o name) + kubectl patch ${FLEXIBLE_SERVER_NAME} -p '{"metadata":{"finalizers":null}}' --type=merge || true + kubectl delete flexibleserver.dbforpostgresql.azure.upbound.io -l crossplane.io/claim-name=${CLAIM_NAME} --force --grace-period=0 || true + ``` \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index c184340..24450f0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -115,8 +115,16 @@ nav: - usecases/azure/packages/mongodb/crossplane/consume-package.md - Prerequisites: - usecases/azure/prerequisites/index.md + - Multicloud: + - Packages: + - Postgresql with Crossplane: + - usecases/multicloud/packages/psql/index.md + - usecases/multicloud/packages/psql/create.md - Crossplane: - crossplane/index.md - Providers: - crossplane/providers/aws.md - crossplane/providers/azure.md + - crossplane/providers/terraform.md + - crossplane/providers/helm.md + - crossplane/providers/kubernetes.md From 80b55b5cd23243999535de42405a05702f41d3b2 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Fri, 20 Jan 2023 14:58:30 +0100 Subject: [PATCH 27/41] fix branch name for publish workflow --- .github/workflows/publish-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index a7e7352..51d005b 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -8,7 +8,7 @@ on: push: branches: - main - - gha-carvel + - develop paths: - 'packages/*/*/*/**' From 42c1cb25aa9fb450c0809ffc46e458c25cbf4624 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Mon, 30 Jan 2023 18:16:46 +0100 Subject: [PATCH 28/41] add explicit package write permissions --- .github/workflows/publish-docs.yml | 3 +++ .github/workflows/publish-packages.yml | 21 ++++----------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 967fe3b..0c22506 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -1,5 +1,8 @@ name: Deploy gh-pages +permissions: + pages: write + on: push: branches: diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index a9ee4f2..3ff7c4c 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -4,6 +4,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: + packages: write + contents: write + on: push: branches: @@ -21,22 +25,7 @@ on: jobs: - parameters: - runs-on: ubuntu-latest - - outputs: - packages_basedir: ${{ steps.params.outputs.packages_basedir}} - - steps: - - - name: Set parameters - id: params - run: | - echo "packages_basedir=${PACKAGES_BASEDIR}" >> $GITHUB_OUTPUT - list-packages: - needs: - - parameters uses: ./.github/workflows/reusable-list-packages.yml with: basedir: packages @@ -49,7 +38,6 @@ jobs: crossplane-publish: needs: - - parameters - bump-version - list-packages if: needs.list-packages.outputs.crossplane_publish == 'true' @@ -66,7 +54,6 @@ jobs: carvel-publish-package: needs: - - parameters - bump-version - list-packages if: needs.list-packages.outputs.carvel_publish == 'true' From f2b41373c82ebf7499af9981e9614f0ea56c58bf Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Thu, 2 Feb 2023 09:20:49 +0100 Subject: [PATCH 29/41] check out the head branch on pr events when publishing carvel repo --- .github/workflows/reusable-carvel-publish-repo.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable-carvel-publish-repo.yml b/.github/workflows/reusable-carvel-publish-repo.yml index 1d950d6..18ff4e7 100644 --- a/.github/workflows/reusable-carvel-publish-repo.yml +++ b/.github/workflows/reusable-carvel-publish-repo.yml @@ -75,11 +75,17 @@ jobs: sleep 5 && git fetch --all - echo "CURRENT_BRANCH=$(git branch --show-current)" >> $GITHUB_ENV + # get the last token splitting by / to get the branch name, as in refs/heads/main + echo "CURRENT_BRANCH=$(grep -oE '[^/]+$' <<<$REF)" | tee -a $GITHUB_ENV + env: + REF: ${{ github.event_name == 'pull_request' && github.head_ref || github.ref }} - name: Create branch id: branch run: | + # make sure the head branch is checked out on pull_request events + git checkout ${CURRENT_BRANCH} + # create new branch git checkout -b ${REPO_BRANCH_NAME} @@ -92,7 +98,7 @@ jobs: BRANCHES="${BRANCHES} ${PACKAGE_BRANCH}" git rebase origin/${PACKAGE_BRANCH} done - trap "git push --delete origin ${BRANCHES}" EXIT + trap "tree -a repository ; git push --delete origin ${BRANCHES}" EXIT # prepare a proper commit message COMMIT_MESSAGE="$(git log --pretty=format:'%s' ${CURRENT_BRANCH}..HEAD)" From ef0d2316135daa8dd8df587281a94551b357ae1c Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Tue, 24 Jan 2023 16:31:59 +0100 Subject: [PATCH 30/41] fix pipelines and crossplane scripts --- .github/workflows/build-docs.yml | 6 +-- .github/workflows/publish-carvel-repo.yml | 32 ------------- .github/workflows/publish-docs.yml | 6 +-- .github/workflows/publish-packages.yml | 8 ++-- .../workflows/reusable-crossplane-publish.yml | 9 +++- .../workflows/reusable-crossplane-test.yml | 25 ++++++++--- ...kip => carvel-e2e-aws-elasticache.skip.sh} | 0 ...crossplane-azure-provider-create-secret.sh | 10 ----- scripts/crossplane-e2e-azure-mongodb.sh | 35 ++++++++------- .../app-overlay.ytt.yml | 28 ++++++++++++ .../claim-instance.sh | 21 +++++---- .../crossplane-e2e-azure-mongodb/cleanup.sh | 17 +++++-- .../install-package.sh | 36 +++++---------- scripts/crossplane-e2e-azure-mongodb/test.sh | 32 +++++++++---- .../crossplane-e2e-install-azure-provider.sh | 24 ---------- .../crossplane-e2e-install-helm-provider.sh | 18 -------- .../crossplane-e2e-install-k8s-provider.sh | 18 -------- scripts/crossplane-e2e-install-tf-provider.sh | 13 ------ ...=> crossplane-e2e-multicloud-psql.sh.skip} | 16 +++---- scripts/crossplane-install-azure-provider.sh | 45 +++++++++++++++++++ scripts/crossplane-install-helm-provider.sh | 22 +++++++++ scripts/crossplane-install-k8s-provider.sh | 22 +++++++++ scripts/crossplane-install-tf-provider.sh | 8 ++++ scripts/crossplane-install-uxp.sh | 1 - scripts/kind-cluster.sh | 3 -- 25 files changed, 250 insertions(+), 205 deletions(-) delete mode 100644 .github/workflows/publish-carvel-repo.yml rename scripts/{carvel-e2e-aws-elasticache.sh.skip => carvel-e2e-aws-elasticache.skip.sh} (100%) delete mode 100755 scripts/crossplane-azure-provider-create-secret.sh create mode 100644 scripts/crossplane-e2e-azure-mongodb/app-overlay.ytt.yml delete mode 100755 scripts/crossplane-e2e-install-azure-provider.sh delete mode 100755 scripts/crossplane-e2e-install-helm-provider.sh delete mode 100755 scripts/crossplane-e2e-install-k8s-provider.sh delete mode 100755 scripts/crossplane-e2e-install-tf-provider.sh rename scripts/{crossplane-e2e-multicloud-psql.sh => crossplane-e2e-multicloud-psql.sh.skip} (78%) create mode 100755 scripts/crossplane-install-azure-provider.sh create mode 100755 scripts/crossplane-install-helm-provider.sh create mode 100755 scripts/crossplane-install-k8s-provider.sh create mode 100755 scripts/crossplane-install-tf-provider.sh delete mode 100755 scripts/kind-cluster.sh diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index af80d60..162597f 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -1,4 +1,4 @@ -name: Deploy gh-pages +name: Build docs on: pull_request: @@ -18,12 +18,12 @@ jobs: fetch-depth: '0' - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.x - name: Install Python requirements run: pip install -r requirements.txt - - name: Deploy GitHub Pages + - name: Build documentation run: mkdocs build diff --git a/.github/workflows/publish-carvel-repo.yml b/.github/workflows/publish-carvel-repo.yml deleted file mode 100644 index 2994c15..0000000 --- a/.github/workflows/publish-carvel-repo.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Publish Carvel repository - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: - - main - paths: - - 'repository/packages/*/*.yml' - - pull_request: - types: [opened, ready_for_review, reopened, synchronize] - branches: - - main - - gha-carvel - paths: - - 'repository/packages/*/*.yml' - -jobs: - - bump-version: - uses: ./.github/workflows/reusable-bump-version.yml - - carvel-repo-publish: - needs: - - bump-version - uses: ./.github/workflows/reusable-carvel-repo-publish.yml - with: - version: ${{ needs.bump-version.outputs.version }} diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 0c22506..f97a190 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -1,4 +1,4 @@ -name: Deploy gh-pages +name: Publish docs permissions: pages: write @@ -21,12 +21,12 @@ jobs: fetch-depth: '0' - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.x - name: Install Python requirements run: pip install -r requirements.txt - - name: Deploy GitHub Pages + - name: Build documentation and deploy GitHub Pages run: mkdocs gh-deploy -b gh-pages diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index 3ff7c4c..79ff317 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -50,7 +50,9 @@ jobs: package_version: ${{ needs.bump-version.outputs.version }} packages_basedir: ${{ needs.list-packages.outputs.basedir }} package_path: ${{ matrix.path }} - run-test: ${{ github.event_name == 'pull_request' }} + run-test: true + # run-test: ${{ github.event_name == 'pull_request' }} + secrets: inherit carvel-publish-package: needs: @@ -80,10 +82,10 @@ jobs: package_version: ${{ needs.bump-version.outputs.version }} repo_version: ${{ needs.bump-version.outputs.version }} prepare_repo_pr: ${{ needs.bump-version.outputs.is_prerelease == 'false' }} - release: true + release: ${{ needs.bump-version.outputs.is_prerelease == 'true' }} carvel-test: - if: needs.list-packages.outputs.carvel_publish == 'true' && github.event_name == 'pull_request' + if: github.event_name == 'pull_request' uses: ./.github/workflows/reusable-carvel-test.yml needs: - list-packages diff --git a/.github/workflows/reusable-crossplane-publish.yml b/.github/workflows/reusable-crossplane-publish.yml index 704ab5e..eb45f49 100644 --- a/.github/workflows/reusable-crossplane-publish.yml +++ b/.github/workflows/reusable-crossplane-publish.yml @@ -31,6 +31,10 @@ on: package_registry: value: ${{ jobs.publish.outputs.package_registry }} + secrets: + AZURE_CONFIG: + required: false + jobs: publish: @@ -55,7 +59,7 @@ jobs: with: ref: ${{ inputs.package_version }} - - name: Install UXP + - name: Install up CLI run: | curl -sL "https://cli.upbound.io" | sh sudo mv up /usr/local/bin/ @@ -68,7 +72,7 @@ jobs: only: ytt - name: Log in to ${{ env.PACKAGE_REGISTRY }} - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 # v1.10.0 + uses: docker/login-action@v2.1.0 with: registry: ${{ env.PACKAGE_REGISTRY }} username: ${{ github.actor }} @@ -93,3 +97,4 @@ jobs: package_provider: ${{ inputs.package_provider }} package_registry: ${{ needs.publish.outputs.package_registry }} package_repository: ${{ needs.publish.outputs.package_repository }} + secrets: inherit diff --git a/.github/workflows/reusable-crossplane-test.yml b/.github/workflows/reusable-crossplane-test.yml index 2a81a7f..ab8b591 100644 --- a/.github/workflows/reusable-crossplane-test.yml +++ b/.github/workflows/reusable-crossplane-test.yml @@ -28,6 +28,9 @@ on: type: string default: v0.16.0 + secrets: + AZURE_CONFIG: + required: false jobs: test: @@ -36,13 +39,25 @@ jobs: CROSSPLANE_NAMESPACE: upbound-system CONFIG_VERSION: ${{ inputs.package_version }} CONFIG_IMAGE: ${{ inputs.package_registry }}/${{ inputs.package_repository }} + PACKAGE_NAME: ${{ inputs.package_name }} steps: + - name: Get runner info + run: | + uname -a + top -bn1 + - name: Checkout uses: actions/checkout@v2 - - name: Install UXP + - name: Install Carvel tools + uses: vmware-tanzu/carvel-setup-action@v1 + with: + only: ytt + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install up CLI run: | curl -sL "https://cli.upbound.io" | sh sudo mv up /usr/local/bin/ @@ -63,11 +78,11 @@ jobs: kubectl cluster-info kubectl get storageclass standard - - name: Create Azure Config Secret - if: inputs.package_provider == 'azure' + - name: Install Universal Crossplane run: | - kubectl create namespace ${CROSSPLANE_NAMESPACE} - kubectl create secret generic azure-secret -n ${CROSSPLANE_NAMESPACE} --from-literal=creds='${{ secrets.AZURE_CONFIG }}' + ./scripts/crossplane-install-uxp.sh ${UXP_VERSION} + env: + UXP_VERSION: v1.10.1-up.1 - name: Test Crossplane package run: | diff --git a/scripts/carvel-e2e-aws-elasticache.sh.skip b/scripts/carvel-e2e-aws-elasticache.skip.sh similarity index 100% rename from scripts/carvel-e2e-aws-elasticache.sh.skip rename to scripts/carvel-e2e-aws-elasticache.skip.sh diff --git a/scripts/crossplane-azure-provider-create-secret.sh b/scripts/crossplane-azure-provider-create-secret.sh deleted file mode 100755 index 3e16ffb..0000000 --- a/scripts/crossplane-azure-provider-create-secret.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -SECRET_NAME=$1 -CREDS=$2 -CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} - -if [ ! -x ${CREDS} ]; then - echo ">> Create Azure Config Secret - secret name=${SECRET_NAME}" - kubectl create secret generic ${SECRET_NAME} -n ${CROSSPLANE_NAMESPACE} --from-literal=creds=${CREDS} -fi diff --git a/scripts/crossplane-e2e-azure-mongodb.sh b/scripts/crossplane-e2e-azure-mongodb.sh index 927197b..8347d18 100755 --- a/scripts/crossplane-e2e-azure-mongodb.sh +++ b/scripts/crossplane-e2e-azure-mongodb.sh @@ -1,31 +1,32 @@ #!/usr/bin/env bash +set -euo pipefail + +trap "top -b -1 -n 1" EXIT + echo ">> Local Test" -export CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} -AZURE_CREDS_SECRET_NAME=${AZURE_CREDS_SECRET_NAME:-"azure-secret"} -UXP_VERSION=${UXP_VERSION:-"v1.10.1-up.1"} -CONFIG_NAME=${CONFIG_NAME:-"trp-azure-mongodb"} -CONFIG_IMAGE=${CONFIG_IMAGE:-"ghcr.io/vmware-tanzu-labs/tap-reference-packages-azure/crossplane-mongodb"} -CONFIG_VERSION=${CONFIG_VERSION:-"0.23.1-beta.0"} -CLAIM_NAME=${CLAIM_NAME:-"trp-cosmosdb-mongo-08"} -TEST_APP_NAME=${TEST_APP_NAME:-"spring-boot-mongo"} +SCRIPT_FOLDER=$(basename $0 .sh) -kubectl create namespace ${CROSSPLANE_NAMESPACE} || true +# SCRIPT_COMMON=${SCRIPT_FOLDER}/common.sh +# [ -x ${SCRIPT_COMMON} ] && source ${SCRIPT_COMMON} -pushd $(dirname $0) +[ -z "${CROSSPLANE_NAMESPACE:-}" ] && ( echo "The CROSSPLANE_NAMESPACE environment variable must be defined" ; exit 1 ) -# Requires AZURE_CONFIG to contain an Azure API credential config (JSON format) -./crossplane-azure-provider-create-secret.sh ${AZURE_CREDS_SECRET_NAME} "${AZURE_CONFIG}" +kubectl create namespace ${CROSSPLANE_NAMESPACE} || true -./crossplane-install-uxp.sh ${UXP_VERSION} +pushd $(dirname $0) -./crossplane-e2e-mongodb/crossplane-install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} ${AZURE_CREDS_SECRET_NAME} +# install provider as well as its ProviderConfig only if the INSTALL_PROVIDER environment variable is not empty +[ -z "${INSTALL_PROVIDER:-}" ] || ./crossplane-install-azure-provider.sh -./crossplane-e2e-mongodb/crossplane-claim-instance.sh ${CLAIM_NAME} +# install the Crossplane configuration +./${SCRIPT_FOLDER}/install-package.sh -./crossplane-e2e-mongodb/crossplane-test.sh ${TEST_APP_NAME} +# create the Crossplane claim +./${SCRIPT_FOLDER}/claim-instance.sh -./crossplane-e2e-mongodb/crossplane-test.sh ${TEST_APP_NAME} +# deploy application and test +./${SCRIPT_FOLDER}/test.sh popd diff --git a/scripts/crossplane-e2e-azure-mongodb/app-overlay.ytt.yml b/scripts/crossplane-e2e-azure-mongodb/app-overlay.ytt.yml new file mode 100644 index 0000000..dcb05d8 --- /dev/null +++ b/scripts/crossplane-e2e-azure-mongodb/app-overlay.ytt.yml @@ -0,0 +1,28 @@ +#@ load("@ytt:overlay", "overlay") +#@ load("@ytt:data", "data") + +#@ name = data.values.name + +#@overlay/match by=overlay.all, expects="1+" +--- +metadata: + name: #@ name + labels: + app.kubernetes.io/name: #@ name +spec: + selector: + matchLabels: + app.kubernetes.io/name: #@ name + template: + metadata: + labels: + app.kubernetes.io/name: #@ name + spec: + volumes: + #@overlay/match by="name" + - name: secret-volume + projected: + sources: + #@overlay/match by=overlay.all, expects="1+" + - secret: + name: #@ data.values.secret diff --git a/scripts/crossplane-e2e-azure-mongodb/claim-instance.sh b/scripts/crossplane-e2e-azure-mongodb/claim-instance.sh index 28f25d5..1d40720 100755 --- a/scripts/crossplane-e2e-azure-mongodb/claim-instance.sh +++ b/scripts/crossplane-e2e-azure-mongodb/claim-instance.sh @@ -1,10 +1,17 @@ #!/usr/bin/env bash -CLAIM_NAME=$1 -CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} +set -euo pipefail +CLAIM_NAME=${1:-${CLAIM_NAME:-}} +[ -z "${CLAIM_NAME:-}" ] && ( echo "The CLAIM_NAME environment variable must be defined" ; exit 1 ) +[ -z "${CROSSPLANE_NAMESPACE:-}" ] && ( echo "The CROSSPLANE_NAMESPACE environment variable must be defined" ; exit 1 ) + +echo +kubectl get xmongodbinstances.azure.ref.services.apps.tanzu.vmware.com,mongodbinstances.azure.ref.services.apps.tanzu.vmware.com + +echo echo ">> Claiming a MongoDBInstance" -cat <> Installing Test Application" -kubectl apply -f https://raw.githubusercontent.com/joostvdg/spring-boot-mongo/main/kubernetes/raw/deployment.yaml -kubectl get deployment +kubectl get xmongodbinstance,mongodbinstance echo ">> Showing Secrets (1)" kubectl get secret -n ${CROSSPLANE_NAMESPACE} diff --git a/scripts/crossplane-e2e-azure-mongodb/cleanup.sh b/scripts/crossplane-e2e-azure-mongodb/cleanup.sh index c71fe5d..cd9345b 100755 --- a/scripts/crossplane-e2e-azure-mongodb/cleanup.sh +++ b/scripts/crossplane-e2e-azure-mongodb/cleanup.sh @@ -1,8 +1,14 @@ #!/usr/bin/env bash -CLAIM_NAME=${CLAIM_NAME:-"trp-cosmosdb-mongo-08"} +set -euo pipefail + +[ -z "${CROSSPLANE_NAMESPACE:-}" ] && ( echo "The CROSSPLANE_NAMESPACE environment variable must be defined" ; exit 1 ) +[ -z "${CLAIM_NAME:-}" ] && ( echo "The CLAIM_NAME environment variable must be defined" ; exit 1 ) +[ -z "${CONFIG_NAME:-}" ] && ( echo "The CONFIG_NAME environment variable must be defined" ; exit 1 ) + TEST_APP_NAME=${TEST_APP_NAME:-"spring-boot-mongo"} -CONFIG_NAME=${CONFIG_NAME:-"trp-azure-mongodb"} +PROVIDER_NAME="upbound-provider-azure" +AZURE_CONFIG_SECRET_NAME="azure-secret" echo ">> Cleaning Up Resources" echo " > WARNING -> This can take a while (~10 minutes), as it waits for Azure to delete the resources!" @@ -14,4 +20,9 @@ kubectl delete MongoCollection -l crossplane.io/claim-name=${CLAIM_NAME} || true kubectl delete Account -l crossplane.io/claim-name=${CLAIM_NAME} || true kubectl delete ResourceGroup -l crossplane.io/claim-name=${CLAIM_NAME} || true kubectl delete configuration ${CONFIG_NAME} || true -kubectl delete providerconfig.azure.upbound.io default || true + +[ -z "${INSTALL_PROVIDER:-}" ] || ( + kubectl delete providerconfig.azure.upbound.io default || true + kubectl delete secret ${AZURE_CONFIG_SECRET_NAME} -n ${CROSSPLANE_NAMESPACE} || true + kubectl delete providers.pkg.crossplane.io ${PROVIDER_NAME} || true +) diff --git a/scripts/crossplane-e2e-azure-mongodb/install-package.sh b/scripts/crossplane-e2e-azure-mongodb/install-package.sh index be41cd1..a7047e3 100755 --- a/scripts/crossplane-e2e-azure-mongodb/install-package.sh +++ b/scripts/crossplane-e2e-azure-mongodb/install-package.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash -CONFIG_NAME=$1 -CONFIG_IMAGE=$2 -CONFIG_VERSION=$3 -AZURE_CONFIG_SECRET_NAME=$4 -CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} +set -euo pipefail + +CONFIG_IMAGE=${1:-${CONFIG_IMAGE:-}} +CONFIG_VERSION=${2:-${CONFIG_VERSION:-}} +[ -z "${CONFIG_IMAGE:-}" ] && ( echo "The CONFIG_IMAGE environment variable must be defined" ; exit 1 ) +[ -z "${CONFIG_VERSION:-}" ] && ( echo "The CONFIG_VERSION environment variable must be defined" ; exit 1 ) echo ">> Installing Crossplane Package via Configuration CR" cat <> Create Azure Provider Config" -cat <> Installing Test Application" +MANIFEST="https://raw.githubusercontent.com/joostvdg/spring-boot-mongo/main/kubernetes/raw/deployment.yaml" +curl -sSfL ${MANIFEST} | ytt -f - -f app-overlay.ytt.yml -v name=${TEST_APP_NAME} -v secret=${CLAIM_NAME} | kubectl apply -n default -f - +kubectl get deployment echo ">> Waiting on Test Application: ${TEST_APP_NAME}" kubectl get pod -kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=${TEST_APP_NAME} --timeout=60s +kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=${TEST_APP_NAME} --timeout=120s echo ">> Starting Port Forward" kubectl port-forward deployment/${TEST_APP_NAME} 8080 & @@ -13,13 +22,18 @@ PORT_FORWARD_PID=$! sleep 10 echo ">> Testing Application" -curl -s "http://localhost:8080" -curl --header "Content-Type: application/json" --request POST --data '{"name":"Piet"}' http://localhost:8080/create -curl --header "Content-Type: application/json" --request POST --data '{"name":"Andrea"}' http://localhost:8080/create -HTTP_RESULT=$(curl -s "http://localhost:8080") -[ $(jq 'map(select(.name =="Piet")) | length'<<<$HTTP_RESULT) -eq 1 ] -[ $(jq 'map(select(.name =="Andrea")) | length'<<<$HTTP_RESULT) -eq 1 ] +curl -sSfL "http://localhost:8080" && echo +echo -n ">> Writing record: " && curl -sSfL --header "Content-Type: application/json" --request POST --data '{"name":"Piet"}' http://localhost:8080/create && echo +echo -n ">> Writing record: " && curl -sSfL --header "Content-Type: application/json" --request POST --data '{"name":"Andrea"}' http://localhost:8080/create && echo + +HTTP_RESULT=$(curl -sSfL "http://localhost:8080") +[ $(jq 'map(select(.name == "Piet")) | length'<<<$HTTP_RESULT) -eq 1 ] +[ $(jq 'map(select(.name == "Andrea")) | length'<<<$HTTP_RESULT) -eq 1 ] + +echo "TEST PASSED" echo ">> Killing Port Forward" echo " > PORT_FORWARD_PID=$PORT_FORWARD_PID" -kill -9 $PORT_FORWARD_PID \ No newline at end of file +kill -9 $PORT_FORWARD_PID + +popd diff --git a/scripts/crossplane-e2e-install-azure-provider.sh b/scripts/crossplane-e2e-install-azure-provider.sh deleted file mode 100755 index 147b22c..0000000 --- a/scripts/crossplane-e2e-install-azure-provider.sh +++ /dev/null @@ -1,24 +0,0 @@ -CROSSPLANE_NAMESPACE=$1 -AZURE_CONFIG_SECRET_NAME=$2 - -up controlplane provider install xpkg.upbound.io/upbound/provider-azure:v0.18.1 || true - -kubectl wait --for=condition="Healthy" providers.pkg.crossplane.io upbound-provider-azure - - -echo ">> Create Azure Provider Config" -cat < Installing required providers" trap $(dirname $0)/crossplane-e2e-multicloud-psql/cleanup.sh EXIT -./crossplane-e2e-install-azure-provider.sh ${CROSSPLANE_NAMESPACE} ${AZURE_CREDS_SECRET_NAME} -./crossplane-e2e-install-helm-provider.sh -./crossplane-e2e-install-k8s-provider.sh -./crossplane-e2e-install-tf-provider.sh +./crossplane-install-azure-provider.sh ${CROSSPLANE_NAMESPACE} ${AZURE_CREDS_SECRET_NAME} +./crossplane-install-helm-provider.sh +./crossplane-install-k8s-provider.sh +./crossplane-install-tf-provider.sh ./crossplane-e2e-multicloud-psql/install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} ./crossplane-e2e-multicloud-psql/claim-helm-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} @@ -40,10 +40,10 @@ trap $(dirname $0)/crossplane-e2e-multicloud-psql/cleanup.sh EXIT sleep 5 -./crossplane-e2e-install-azure-provider.sh ${CROSSPLANE_NAMESPACE} ${AZURE_CREDS_SECRET_NAME} -./crossplane-e2e-install-helm-provider.sh -./crossplane-e2e-install-k8s-provider.sh -./crossplane-e2e-install-tf-provider.sh +./crossplane-install-azure-provider.sh ${CROSSPLANE_NAMESPACE} ${AZURE_CREDS_SECRET_NAME} +./crossplane-install-helm-provider.sh +./crossplane-install-k8s-provider.sh +./crossplane-install-tf-provider.sh ./crossplane-e2e-multicloud-psql/install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} ./crossplane-e2e-multicloud-psql/claim-azure-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} diff --git a/scripts/crossplane-install-azure-provider.sh b/scripts/crossplane-install-azure-provider.sh new file mode 100755 index 0000000..476d2dd --- /dev/null +++ b/scripts/crossplane-install-azure-provider.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -euo pipefail + +[ -z "${CROSSPLANE_NAMESPACE:-}" ] && ( echo "The CROSSPLANE_NAMESPACE environment variable must be defined" ; exit 1 ) +[ -z "${AZURE_CONFIG:-}" ] && ( echo "The AZURE_CONFIG environment variable must be defined" ; exit 1 ) + +PROVIDER_NAME="upbound-provider-azure" +AZURE_CONFIG_SECRET_NAME="azure-secret" + +kubectl create namespace ${CROSSPLANE_NAMESPACE} || true + +echo ">> Install ${PROVIDER_NAME} provider" +up controlplane provider install xpkg.upbound.io/upbound/provider-azure:v0.18.1 --name ${PROVIDER_NAME} || true + +kubectl wait --for=condition="Healthy" providers.pkg.crossplane.io ${PROVIDER_NAME} +kubectl wait --for=condition=Available=True apiservices.apiregistration.k8s.io v1beta1.azure.upbound.io + +echo ">> Create Azure Config Secret - secret name=${AZURE_CONFIG_SECRET_NAME}" +kubectl delete secret ${AZURE_CONFIG_SECRET_NAME} -n ${CROSSPLANE_NAMESPACE} || true +kubectl create secret generic ${AZURE_CONFIG_SECRET_NAME} -n ${CROSSPLANE_NAMESPACE} --from-literal=creds="${AZURE_CONFIG}" + +echo ">> Create Azure Provider Config" +cat <> Installing UXP - Universal Crossplane" diff --git a/scripts/kind-cluster.sh b/scripts/kind-cluster.sh deleted file mode 100755 index 7f498fd..0000000 --- a/scripts/kind-cluster.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -kind create cluster --name test --wait 5m From 4a79500005fefe2dd0659982a290720fd20dabf5 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Wed, 25 Jan 2023 15:00:20 +0100 Subject: [PATCH 31/41] add azure mongodb package to packages dir --- .../mongodb-32-claim-example.yml | 22 ++++ .../mongodb-3x-claim-example.yml | 21 ++++ .../mongodb-4x-claim-example.yml | 22 ++++ .../mongodb/ytt/composition.ytt.yml | 118 ++++++++++++++++++ .../crossplane/mongodb/ytt/crossplane.ytt.yml | 12 ++ .../crossplane/mongodb/ytt/definition.ytt.yml | 76 +++++++++++ .../crossplane/mongodb/ytt/schema.ytt.yml | 34 +++++ .../crossplane/psql/ytt/shared.lib.yml | 3 +- 8 files changed, 307 insertions(+), 1 deletion(-) create mode 100644 packages/azure/crossplane/mongodb/claim-examples/mongodb-32-claim-example.yml create mode 100644 packages/azure/crossplane/mongodb/claim-examples/mongodb-3x-claim-example.yml create mode 100644 packages/azure/crossplane/mongodb/claim-examples/mongodb-4x-claim-example.yml create mode 100644 packages/azure/crossplane/mongodb/ytt/composition.ytt.yml create mode 100644 packages/azure/crossplane/mongodb/ytt/crossplane.ytt.yml create mode 100644 packages/azure/crossplane/mongodb/ytt/definition.ytt.yml create mode 100644 packages/azure/crossplane/mongodb/ytt/schema.ytt.yml diff --git a/packages/azure/crossplane/mongodb/claim-examples/mongodb-32-claim-example.yml b/packages/azure/crossplane/mongodb/claim-examples/mongodb-32-claim-example.yml new file mode 100644 index 0000000..d3d9990 --- /dev/null +++ b/packages/azure/crossplane/mongodb/claim-examples/mongodb-32-claim-example.yml @@ -0,0 +1,22 @@ +apiVersion: azure.ref.services.apps.tanzu.vmware.com/v1alpha1 +kind: MongoDBInstance +metadata: + namespace: default + name: trp-cosmosdb-mongo-09 +spec: + compositionSelector: + matchLabels: + database: mongodb + parameters: + location: "West Europe" + mongodbVersion: "3.2" + capabilities: + - name: "EnableMongo" + - name: "mongoEnableDocLevelTTL" + publishConnectionDetailsTo: + name: trp-cosmosdb-mongo-bindable-09 + configRef: + name: default + metadata: + labels: + services.apps.tanzu.vmware.com/class: azure-mongodb \ No newline at end of file diff --git a/packages/azure/crossplane/mongodb/claim-examples/mongodb-3x-claim-example.yml b/packages/azure/crossplane/mongodb/claim-examples/mongodb-3x-claim-example.yml new file mode 100644 index 0000000..943e9ac --- /dev/null +++ b/packages/azure/crossplane/mongodb/claim-examples/mongodb-3x-claim-example.yml @@ -0,0 +1,21 @@ +apiVersion: azure.ref.services.apps.tanzu.vmware.com/v1alpha1 +kind: MongoDBInstance +metadata: + namespace: default + name: trp-cosmosdb-mongo-08 +spec: + compositionSelector: + matchLabels: + database: mongodb + parameters: + location: "West Europe" + capabilities: + - name: "EnableMongo" + - name: "mongoEnableDocLevelTTL" + publishConnectionDetailsTo: + name: trp-cosmosdb-mongo-bindable-08 + configRef: + name: default + metadata: + labels: + services.apps.tanzu.vmware.com/class: azure-mongodb \ No newline at end of file diff --git a/packages/azure/crossplane/mongodb/claim-examples/mongodb-4x-claim-example.yml b/packages/azure/crossplane/mongodb/claim-examples/mongodb-4x-claim-example.yml new file mode 100644 index 0000000..f2e2e54 --- /dev/null +++ b/packages/azure/crossplane/mongodb/claim-examples/mongodb-4x-claim-example.yml @@ -0,0 +1,22 @@ +apiVersion: azure.ref.services.apps.tanzu.vmware.com/v1alpha1 +kind: MongoDBInstance +metadata: + namespace: default + name: trp-cosmosdb-mongo-09 +spec: + compositionSelector: + matchLabels: + database: mongodb + parameters: + location: "West Europe" + mongodbVersion: "4.2" + capabilities: + - name: "EnableMongo" + - name: "mongoEnableDocLevelTTL" + publishConnectionDetailsTo: + name: trp-cosmosdb-mongo-bindable-09 + configRef: + name: default + metadata: + labels: + services.apps.tanzu.vmware.com/class: azure-mongodb \ No newline at end of file diff --git a/packages/azure/crossplane/mongodb/ytt/composition.ytt.yml b/packages/azure/crossplane/mongodb/ytt/composition.ytt.yml new file mode 100644 index 0000000..3010c9d --- /dev/null +++ b/packages/azure/crossplane/mongodb/ytt/composition.ytt.yml @@ -0,0 +1,118 @@ +#@ load("@ytt:data", "data") +--- +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: #@ (data.values.xrd.claimNames.kind).lower() + labels: + crossplane.io/xrd: #@ data.values.xrd.names.plural + "." + data.values.xrd.group + provider: #@ data.values.provider.name + database: #@ data.values.cloudServiceBindingType +spec: + publishConnectionDetailsWithStoreConfigRef: + name: #@ data.values.storeConfig.name + compositeTypeRef: + apiVersion: #@ data.values.xrd.group + "/" + data.values.xrd.version + kind: #@ data.values.xrd.names.kind + resources: + - name: resourcegroup + base: + apiVersion: azure.upbound.io/v1beta1 + kind: ResourceGroup + spec: {} + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.location + - name: account + base: + apiVersion: cosmosdb.azure.upbound.io/v1beta1 + kind: Account + spec: + forProvider: + consistencyPolicy: + - consistencyLevel: Strong + geoLocation: + - failoverPriority: 0 + kind: MongoDB + offerType: Standard + resourceGroupNameSelector: + matchControllerRef: true + providerConfigRef: + name: #@ data.values.provider.configRef + writeConnectionSecretToRef: + namespace: #@ data.values.crossplane.namespace + patches: + - type: FromCompositeFieldPath + fromFieldPath: metadata.uid + toFieldPath: spec.writeConnectionSecretToRef.name + transforms: + - type: string + string: + fmt: "%s-mongodb" + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.capabilities + toFieldPath: spec.forProvider.capabilities + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.mongodbVersion + toFieldPath: spec.forProvider.mongoServerVersion + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.location + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.location + toFieldPath: spec.forProvider.geoLocation[0].location + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.providerConfig + toFieldPath: spec.providerConfigRef.name + - type: ToCompositeFieldPath + fromFieldPath: spec.forProvider.mongoServerVersion + toFieldPath: status.mongodbVersion + - type: ToCompositeFieldPath + fromFieldPath: spec.forProvider.location + toFieldPath: status.location + - type: ToCompositeFieldPath + fromFieldPath: status.atProvider.endpoint + toFieldPath: status.endpoint + connectionDetails: + - name: uri + fromConnectionSecretKey: "attribute.connection_strings.0" + - type: FromValue + name: type + value: #@ data.values.cloudServiceBindingType + - name: mongodatabase + base: + apiVersion: cosmosdb.azure.upbound.io/v1beta1 + kind: MongoDatabase + spec: + forProvider: + accountNameSelector: + matchControllerRef: true + resourceGroupNameSelector: + matchControllerRef: true + providerConfigRef: + name: #@ data.values.provider.configRef + patches: + connectionDetails: + - type: FromFieldPath + name: database + fromFieldPath: metadata.name + - name: mongocollection + base: + apiVersion: cosmosdb.azure.upbound.io/v1beta1 + kind: MongoCollection + spec: + forProvider: + accountNameSelector: + matchControllerRef: true + databaseNameSelector: + matchControllerRef: true + resourceGroupNameSelector: + matchControllerRef: true + defaultTtlSeconds: 777 + index: + - keys: + - _id + unique: true + shardKey: uniqueKey + throughput: 400 diff --git a/packages/azure/crossplane/mongodb/ytt/crossplane.ytt.yml b/packages/azure/crossplane/mongodb/ytt/crossplane.ytt.yml new file mode 100644 index 0000000..1b95d20 --- /dev/null +++ b/packages/azure/crossplane/mongodb/ytt/crossplane.ytt.yml @@ -0,0 +1,12 @@ +#@ load("@ytt:data", "data") +--- +apiVersion: meta.pkg.crossplane.io/v1 +kind: Configuration +metadata: + name: #@ data.values.configurationName + annotations: + provider: #@ data.values.provider.name +spec: + dependsOn: + - provider: #@ data.values.provider.image + version: #@ data.values.provider.version \ No newline at end of file diff --git a/packages/azure/crossplane/mongodb/ytt/definition.ytt.yml b/packages/azure/crossplane/mongodb/ytt/definition.ytt.yml new file mode 100644 index 0000000..95facbd --- /dev/null +++ b/packages/azure/crossplane/mongodb/ytt/definition.ytt.yml @@ -0,0 +1,76 @@ +#@ load("@ytt:data", "data") +--- +apiVersion: apiextensions.crossplane.io/v1 +kind: CompositeResourceDefinition +metadata: + name: #@ data.values.xrd.names.plural + "." + data.values.xrd.group +spec: + group: #@ data.values.xrd.group + names: #@ data.values.xrd.names + claimNames: #@ data.values.xrd.claimNames + connectionSecretKeys: + - database + - uri + - type + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + parameters: + type: object + properties: + location: + type: string + providerConfig: + type: string + default: default + mongodbVersion: + type: string + default: "3.6" + enum: ["4.2", "4.0", "3.6", "3.2"] + capabilities: + type: array + items: + type: object + properties: + name: + type: string + required: + - name + required: + - location + - capabilities + required: + - parameters + status: + type: object + properties: + mongodbVersion: + description: The version of the MongoDB server + type: string + location: + description: The location of the MongoDB server + type: string + endpoint: + description: The endpoint of the MongoDB server + type: string + additionalPrinterColumns: + - name: mongodbVersion + type: string + jsonPath: ".status.mongodbVersion" + - name: location + type: string + jsonPath: ".status.location" + - name: endpoint + type: string + jsonPath: ".status.endpoint" + - name: connection-details + type: string + jsonPath: ".spec.publishConnectionDetailsTo.name" \ No newline at end of file diff --git a/packages/azure/crossplane/mongodb/ytt/schema.ytt.yml b/packages/azure/crossplane/mongodb/ytt/schema.ytt.yml new file mode 100644 index 0000000..508582c --- /dev/null +++ b/packages/azure/crossplane/mongodb/ytt/schema.ytt.yml @@ -0,0 +1,34 @@ +#@data/values-schema +--- + +xrd: + group: azure.ref.services.apps.tanzu.vmware.com + names: + kind: XMongoDBInstance + plural: xmongodbinstances + claimNames: + kind: MongoDBInstance + plural: mongodbinstances + version: v1alpha1 + +configurationName: "azure-mongodb" +cloudServiceBindingType: "mongodb" + +provider: + name: azure + image: xpkg.upbound.io/upbound/provider-azure + version: ">=v0.18.1" + configRef: default + +crossplane: + #@schema/title "CrossplaneNamespace" + #@schema/desc "The namespace where crossplane controller is installed" + namespace: upbound-system + version: '^v1.10' + +#@schema/title "StoreConfig" +#@schema/desc "Details of the StoreConfig" +storeConfig: + #@schema/title "StoreConfig Name" + #@schema/desc "The name of the StoreConfig" + name: "default" diff --git a/packages/multicloud/crossplane/psql/ytt/shared.lib.yml b/packages/multicloud/crossplane/psql/ytt/shared.lib.yml index 39ec00a..7a05beb 100644 --- a/packages/multicloud/crossplane/psql/ytt/shared.lib.yml +++ b/packages/multicloud/crossplane/psql/ytt/shared.lib.yml @@ -29,6 +29,7 @@ patches: toFieldPath: spec.forProvider.manifest.metadata.labels[services.apps.tanzu.vmware.com/location] #@ end +--- #@ def tfProviderConfig(crossplaneNamespace): name: tf-providerconfig base: @@ -64,7 +65,7 @@ patches: toFieldPath: spec.forProvider.manifest.metadata.name #@ end - +--- #@ def tfWorkspace(crossplaneNamespace): name: password base: From 19a074e9ea30bd2d8f80da1d7e969fa43992f1a0 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Tue, 31 Jan 2023 11:31:04 +0100 Subject: [PATCH 32/41] testing with kubectl 1.26.0 and kind 0.17.0 --- .github/workflows/publish-packages.yml | 3 +- .github/workflows/reusable-carvel-test.yml | 10 +++---- .../workflows/reusable-crossplane-test.yml | 28 +++++++++++++------ scripts/crossplane-e2e-azure-mongodb.sh | 2 +- .../install-package.sh | 4 +-- scripts/crossplane-install-azure-provider.sh | 2 ++ 6 files changed, 31 insertions(+), 18 deletions(-) diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index 79ff317..cdbdbbb 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -50,8 +50,7 @@ jobs: package_version: ${{ needs.bump-version.outputs.version }} packages_basedir: ${{ needs.list-packages.outputs.basedir }} package_path: ${{ matrix.path }} - run-test: true - # run-test: ${{ github.event_name == 'pull_request' }} + run-test: ${{ github.event_name == 'pull_request' }} secrets: inherit carvel-publish-package: diff --git a/.github/workflows/reusable-carvel-test.yml b/.github/workflows/reusable-carvel-test.yml index a443c2a..5608eb5 100644 --- a/.github/workflows/reusable-carvel-test.yml +++ b/.github/workflows/reusable-carvel-test.yml @@ -152,18 +152,18 @@ jobs: ${SCRIPT} fi env: - AZURE_CONFIG: ${{ secrets.AZURE_CONFIG }} SCRIPT: ./scripts/carvel-e2e-${{ inputs.package_provider }}-${{ inputs.package_name }}.sh PACKAGE_DIR: ${{ inputs.packages_basedir }}/${{ inputs.package_path }} PRERELEASE: ${{ inputs.package_prerelease }} PULL_REQUEST_NUMBER: ${{ inputs.pull_request_number }} GH_TOKEN: ${{ github.token }} - - name: Cleanup Carvel package + # Azure-specific variable + AZURE_CONFIG: ${{ secrets.AZURE_CONFIG }} + + - name: Cleanup if: always() run: | - if [ -x ${SCRIPT} ]; then - ${SCRIPT} - fi + if [ -x ${SCRIPT} ]; then ${SCRIPT}; fi env: SCRIPT: ./scripts/carvel-e2e-${{ inputs.package_provider }}-${{ inputs.package_name }}/cleanup.sh diff --git a/.github/workflows/reusable-crossplane-test.yml b/.github/workflows/reusable-crossplane-test.yml index ab8b591..27bad9a 100644 --- a/.github/workflows/reusable-crossplane-test.yml +++ b/.github/workflows/reusable-crossplane-test.yml @@ -23,10 +23,10 @@ on: required: true kubernetes_version: type: string - default: v1.24.6 + default: v1.26.0 kind_version: type: string - default: v0.16.0 + default: v0.17.0 secrets: AZURE_CONFIG: @@ -39,7 +39,7 @@ jobs: CROSSPLANE_NAMESPACE: upbound-system CONFIG_VERSION: ${{ inputs.package_version }} CONFIG_IMAGE: ${{ inputs.package_registry }}/${{ inputs.package_repository }} - PACKAGE_NAME: ${{ inputs.package_name }} + UNIQUE_NAME: ${{ inputs.package_name }}-github-${{ github.run_id }}-${{ github.run_attempt }} steps: @@ -64,7 +64,7 @@ jobs: up --version - name: Create k8s Kind Cluster - uses: helm/kind-action@v1.4.0 + uses: helm/kind-action@v1.5.0 with: verbosity: 5 version: ${{ inputs.kind_version }} @@ -85,14 +85,26 @@ jobs: UXP_VERSION: v1.10.1-up.1 - name: Test Crossplane package + id: test run: | - [ -x ${SCRIPT} ] && ${SCRIPT} || true + if [ -x ${SCRIPT} ]; then ${SCRIPT}; fi env: - SCRIPT: ./scripts/crossplane-e2e-${{ inputs.package_name }}.sh + INSTALL_PROVIDER: "true" - - name: Cleanup Crossplane package + SCRIPT: ./scripts/crossplane-e2e-${{ inputs.package_provider }}-${{ inputs.package_name }}.sh + CLAIM_NAME: ${{ env.UNIQUE_NAME }} + TEST_APP_NAME: ${{ env.UNIQUE_NAME }} + CONFIG_NAME: ${{ env.UNIQUE_NAME }} + + # Azure-specific variable + AZURE_CONFIG: ${{ secrets.AZURE_CONFIG }} + + - name: Cleanup if: always() run: | - [ -x ${SCRIPT} ] && ${SCRIPT} || true + if [ -x ${SCRIPT} ]; then ${SCRIPT}; fi env: SCRIPT: ./scripts/crossplane-e2e-${{ inputs.package_name }}/cleanup.sh + CLAIM_NAME: ${{ env.UNIQUE_NAME }} + TEST_APP_NAME: ${{ env.UNIQUE_NAME }} + CONFIG_NAME: ${{ env.UNIQUE_NAME }} diff --git a/scripts/crossplane-e2e-azure-mongodb.sh b/scripts/crossplane-e2e-azure-mongodb.sh index 8347d18..756c117 100755 --- a/scripts/crossplane-e2e-azure-mongodb.sh +++ b/scripts/crossplane-e2e-azure-mongodb.sh @@ -2,7 +2,7 @@ set -euo pipefail -trap "top -b -1 -n 1" EXIT +trap "echo '###ERROR###' ; echo !! ; top -b -1 -n 1" ERR echo ">> Local Test" diff --git a/scripts/crossplane-e2e-azure-mongodb/install-package.sh b/scripts/crossplane-e2e-azure-mongodb/install-package.sh index a7047e3..249f701 100755 --- a/scripts/crossplane-e2e-azure-mongodb/install-package.sh +++ b/scripts/crossplane-e2e-azure-mongodb/install-package.sh @@ -25,6 +25,8 @@ spec: skipDependencyResolution: false EOF +sleep 300 + kubectl wait --for=condition=Healthy --timeout=5m configuration ${CONFIG_NAME} kubectl wait --for=condition=Healthy --timeout=5m configurationrevisions.pkg.crossplane.io -l pkg.crossplane.io/package=${CONFIG_NAME} kubectl get configuration @@ -32,5 +34,3 @@ kubectl describe configurationrevisions.pkg.crossplane.io kubectl wait --for=condition=Available apiservices.apiregistration.k8s.io v1alpha1.azure.ref.services.apps.tanzu.vmware.com echo kubectl api-resources --api-group azure.ref.services.apps.tanzu.vmware.com - -sleep 1 diff --git a/scripts/crossplane-install-azure-provider.sh b/scripts/crossplane-install-azure-provider.sh index 476d2dd..58c7edf 100755 --- a/scripts/crossplane-install-azure-provider.sh +++ b/scripts/crossplane-install-azure-provider.sh @@ -35,6 +35,8 @@ spec: key: creds EOF +sleep 120 + kubectl wait --for=condition=Healthy --timeout=5m providers.pkg.crossplane.io ${PROVIDER_NAME} kubectl -n upbound-system get deployments.apps From 20bc19c94ac0d18b26d757ca91fad4498a8d4ac2 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Mon, 6 Feb 2023 10:08:54 +0100 Subject: [PATCH 33/41] review multicloud scripts --- scripts/crossplane-e2e-azure-mongodb.sh | 5 +- scripts/crossplane-e2e-multicloud-psql.sh | 73 +++++++++++++++++++ .../crossplane-e2e-multicloud-psql.sh.skip | 54 -------------- .../install-package.sh | 16 ++-- 4 files changed, 84 insertions(+), 64 deletions(-) create mode 100755 scripts/crossplane-e2e-multicloud-psql.sh delete mode 100755 scripts/crossplane-e2e-multicloud-psql.sh.skip diff --git a/scripts/crossplane-e2e-azure-mongodb.sh b/scripts/crossplane-e2e-azure-mongodb.sh index 756c117..f4e6ede 100755 --- a/scripts/crossplane-e2e-azure-mongodb.sh +++ b/scripts/crossplane-e2e-azure-mongodb.sh @@ -2,15 +2,12 @@ set -euo pipefail -trap "echo '###ERROR###' ; echo !! ; top -b -1 -n 1" ERR +trap "echo '###ERROR###' ; !! ; top -b -1 -n 1" ERR echo ">> Local Test" SCRIPT_FOLDER=$(basename $0 .sh) -# SCRIPT_COMMON=${SCRIPT_FOLDER}/common.sh -# [ -x ${SCRIPT_COMMON} ] && source ${SCRIPT_COMMON} - [ -z "${CROSSPLANE_NAMESPACE:-}" ] && ( echo "The CROSSPLANE_NAMESPACE environment variable must be defined" ; exit 1 ) kubectl create namespace ${CROSSPLANE_NAMESPACE} || true diff --git a/scripts/crossplane-e2e-multicloud-psql.sh b/scripts/crossplane-e2e-multicloud-psql.sh new file mode 100755 index 0000000..a9a7768 --- /dev/null +++ b/scripts/crossplane-e2e-multicloud-psql.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +set -euo pipefail + +trap "echo '###ERROR###' ; !! ; top -b -1 -n 1" ERR + +echo ">> Local Test" + +SCRIPT_FOLDER=$(basename $0 .sh) + +[ -z "${CROSSPLANE_NAMESPACE:-}" ] && ( echo "The CROSSPLANE_NAMESPACE environment variable must be defined" ; exit 1 ) + +kubectl create namespace ${CROSSPLANE_NAMESPACE} || true + +pushd $(dirname $0) + + +# echo ">> Local Test" + +# export CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} +# AZURE_CREDS_SECRET_NAME=${AZURE_CREDS_SECRET_NAME:-"azure-secret"} +# UXP_VERSION=${UXP_VERSION:-"v1.10.1-up.1"} +# CONFIG_NAME=${CONFIG_NAME:-"trp-multicloud-psql"} +# CONFIG_IMAGE=${CONFIG_IMAGE:-"ghcr.io/vmware-tanzu-labs/trp-azure-psql"} +# CONFIG_VERSION=${CONFIG_VERSION:-"0.0.1-rc-1"} +# CLAIM_NAME=${CLAIM_NAME:-"postgresql-0001"} +# TEST_APP_NAME=${TEST_APP_NAME:-"spring-boot-postgres"} +STORAGE_CLASS=${STORAGE_CLASS:-"default"} + + +# kubectl create namespace ${CROSSPLANE_NAMESPACE} || true + +# pushd $(dirname $0) + +# TODO use this when we also have the Azure version +# Requires AZURE_CONFIG to contain an Azure API credential config (JSON format) +# ./crossplane-azure-provider-create-secret.sh ${AZURE_CREDS_SECRET_NAME} "${AZURE_CONFIG}" + +# ./crossplane-install-uxp.sh ${UXP_VERSION} + + +echo "> Installing required providers" + +# trap $(dirname $0)/crossplane-e2e-multicloud-psql/cleanup.sh EXIT + +# install provider as well as its ProviderConfig only if the INSTALL_PROVIDER environment variable is not empty +[ -z "${INSTALL_PROVIDER:-}" ] || ( + ./crossplane-install-helm-provider.sh + ./crossplane-install-k8s-provider.sh + ./crossplane-install-tf-provider.sh +) + +./${SCRIPT_FOLDER}/install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} +./${SCRIPT_FOLDER}/claim-helm-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} +./${SCRIPT_FOLDER}/test.sh ${TEST_APP_NAME} + +# ./${SCRIPT_FOLDER}/cleanup.sh + +sleep 5 + +[ -z "${INSTALL_PROVIDER:-}" ] || ( + ./crossplane-install-azure-provider.sh + ./crossplane-install-k8s-provider.sh + ./crossplane-install-tf-provider.sh +) + +./${SCRIPT_FOLDER}/install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} +./${SCRIPT_FOLDER}/claim-azure-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} +./${SCRIPT_FOLDER}/test.sh ${TEST_APP_NAME} + +./crossplane-e2e-multicloud-psql/cleanup.sh + +popd diff --git a/scripts/crossplane-e2e-multicloud-psql.sh.skip b/scripts/crossplane-e2e-multicloud-psql.sh.skip deleted file mode 100755 index cba27e6..0000000 --- a/scripts/crossplane-e2e-multicloud-psql.sh.skip +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash - -echo ">> Local Test" - -export CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} -AZURE_CREDS_SECRET_NAME=${AZURE_CREDS_SECRET_NAME:-"azure-secret"} -UXP_VERSION=${UXP_VERSION:-"v1.10.1-up.1"} -CONFIG_NAME=${CONFIG_NAME:-"trp-multicloud-psql"} -CONFIG_IMAGE=${CONFIG_IMAGE:-"ghcr.io/vmware-tanzu-labs/trp-azure-psql"} -CONFIG_VERSION=${CONFIG_VERSION:-"0.0.1-rc-1"} -CLAIM_NAME=${CLAIM_NAME:-"postgresql-0001"} -TEST_APP_NAME=${TEST_APP_NAME:-"spring-boot-postgres"} -STORAGE_CLASS=${STORAGE_CLASS:-"default"} - -kubectl create namespace ${CROSSPLANE_NAMESPACE} || true - -pushd $(dirname $0) - -# TODO use this when we also have the Azure version -# Requires AZURE_CONFIG to contain an Azure API credential config (JSON format) -./crossplane-azure-provider-create-secret.sh ${AZURE_CREDS_SECRET_NAME} "${AZURE_CONFIG}" - -./crossplane-install-uxp.sh ${UXP_VERSION} - - -echo "> Installing required providers" - -trap $(dirname $0)/crossplane-e2e-multicloud-psql/cleanup.sh EXIT - -./crossplane-install-azure-provider.sh ${CROSSPLANE_NAMESPACE} ${AZURE_CREDS_SECRET_NAME} -./crossplane-install-helm-provider.sh -./crossplane-install-k8s-provider.sh -./crossplane-install-tf-provider.sh - -./crossplane-e2e-multicloud-psql/install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} -./crossplane-e2e-multicloud-psql/claim-helm-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} -./crossplane-e2e-multicloud-psql/test.sh ${TEST_APP_NAME} - -./crossplane-e2e-multicloud-psql/cleanup.sh - -sleep 5 - -./crossplane-install-azure-provider.sh ${CROSSPLANE_NAMESPACE} ${AZURE_CREDS_SECRET_NAME} -./crossplane-install-helm-provider.sh -./crossplane-install-k8s-provider.sh -./crossplane-install-tf-provider.sh -./crossplane-e2e-multicloud-psql/install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} - -./crossplane-e2e-multicloud-psql/claim-azure-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} -./crossplane-e2e-multicloud-psql/test.sh ${TEST_APP_NAME} - -./crossplane-e2e-multicloud-psql/cleanup.sh - -popd diff --git a/scripts/crossplane-e2e-multicloud-psql/install-package.sh b/scripts/crossplane-e2e-multicloud-psql/install-package.sh index dd0ded0..b9da69b 100755 --- a/scripts/crossplane-e2e-multicloud-psql/install-package.sh +++ b/scripts/crossplane-e2e-multicloud-psql/install-package.sh @@ -1,9 +1,11 @@ #!/usr/bin/env bash -CONFIG_NAME=$1 -CONFIG_IMAGE=$2 -CONFIG_VERSION=$3 -CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} +CONFIG_NAME=${1:-${CONFIG_NAME:-}} +CONFIG_IMAGE=${2:-${CONFIG_IMAGE:-}} +CONFIG_VERSION=${3:-${CONFIG_VERSION:-}} +[ -z "${CONFIG_NAME:-}" ] && ( echo "The CONFIG_NAME environment variable must be defined" ; exit 1 ) +[ -z "${CONFIG_IMAGE:-}" ] && ( echo "The CONFIG_IMAGE environment variable must be defined" ; exit 1 ) +[ -z "${CONFIG_VERSION:-}" ] && ( echo "The CONFIG_VERSION environment variable must be defined" ; exit 1 ) echo ">> Installing Crossplane Package via Configuration CR" cat < Date: Tue, 24 Jan 2023 16:31:14 +0100 Subject: [PATCH 34/41] write workflows docs --- docs/ci/carvel-crossplane-packages.md | 126 ++++++++++++++++++++++++++ docs/ci/documentation.md | 35 +++++++ docs/ci/index.md | 31 +++++++ mkdocs.yml | 6 +- 4 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 docs/ci/carvel-crossplane-packages.md create mode 100644 docs/ci/documentation.md create mode 100644 docs/ci/index.md diff --git a/docs/ci/carvel-crossplane-packages.md b/docs/ci/carvel-crossplane-packages.md new file mode 100644 index 0000000..4358fce --- /dev/null +++ b/docs/ci/carvel-crossplane-packages.md @@ -0,0 +1,126 @@ +--- +title: Carvel and Crossplane packages +--- + +The code and configuration for the reference packages must be stored in the `packages` folder, +in a structure like the following: + +```text +packages +├── +│ └── +│ └── +│ ├── ... +│ ├── ... +┆ ┆ +``` + +- `PROVIDER`: the provider the package is built for (i.e. `aws`, `azure`, `gcp`, `multicloud`). +- `PACKAGING`: the packaging system used (currently it must be either `carvel` or `crossplane`). +- `NAME`: the package name. + +Multiple packages can be built and tested in parallel via the `publish-packages.yml` workflow, which is represented in the following diagram. + +```mermaid +graph LR + A(List packages) --> B(Bump version) + B --> C + B --> D + + subgraph Crossplane + D(Publish Crossplane packages) --> G(Test Crossplane packages) + end + + subgraph Carvel + C(Publish Carvel packages) --> E(Publish Carvel repository) + E --> F(Test Carvel packages) + end + + click A "#list-packages" + click B "#bump-version" + click C "#publish-carvel-packages" + click E "#publish-carvel-repository" + click F "#test-carvel-packages" + click D "#publish-crossplane-packages" + click G "#test-crossplane-packages" +``` + +This workflow runs upon push to either `main` or `develop` branches as well as pull request events targeting the `main` branch. +Only the files matching the `packages/*/*/*/**` glob pattern can actually trigger the workflow: this reflects the packages structure described before +and prevents changes to files outside actual packages directories to trigger it. + +## List packages + +This job evaluates the differences since the previous commit and builds two lists of packages, for Carvel and Crossplane, to feed the following GitHub Actions matrix jobs with. + +## Bump version + +The repository is tagged with a new version, bumping the latest one by either the major, minor or patch fields. +Which field is increased depends on the git comment having any of the substrings `#major`, `#minor`, `#patch` (default is set to `patch`). +More info [here](https://github.com/anothrNick/github-tag-action/tree/1.57.0). + +## Publish Carvel packages + +The Carvel packages that have been changed from the previous commit are built in parallel and published to GitHub Packages using [Carvel's `kctrl`](https://carvel.dev/kapp-controller/docs/v0.43.2/kctrl-package-authoring/). +`kctrl` also produces packages metadata files that are needed to build Carvel's repository, which are then stored in a separate git branch for each package. + +## Publish Carvel repository + +The packages' metadata files produced in the previous job are collected into one single branch, where `kctrl` is used to author the actual Carvel repository publish it to GitHub Packages in OCI format. +The artefact produced by `kctrl` is a manifest file for the `PackageRepository` resource, useful for quickly deploying the repository on top of a kapp-enabled Kubernetes cluster. +This file is attached to a pre-release, which will then be used in the following job for testing the packages. + +!!! note "" + This Carvel repository is built for the sole purpose of testing the packages, thus a pre-release is created. + The actual repository release is taken care of by a different workflow. + +## Test Carvel packages + +This phase aims at testing the Carvel packages, by spawning one job per package and running them in parallel on separate GitHub Actions runners. +Each job performs the following steps: + +1. [install the Carvel suite](https://carvel.dev/#install) +1. create a [kind cluster](https://kind.sigs.k8s.io/) +1. [install kapp-controller](https://carvel.dev/kapp-controller/docs/v0.43.2/install/) +1. [install secretgen-controller](https://github.com/carvel-dev/secretgen-controller/blob/develop/docs/install.md) +1. install the Carvel repository published in the previous job +1. run a test script that adheres to a specific naming convention (`scripts/carvel-e2e-$PACKAGE_PROVIDER-$PACKAGE_NAME.sh`), in order to: + + 1. install the cloud operator specific for the current package (i.e. [ACK](https://aws-controllers-k8s.github.io/community/docs/community/overview/), [ASO](https://azure.github.io/azure-service-operator/), ...) + 1. deploy the package with sample input data (need to consider also firewall rules to allow network traffic) + 1. deploy an application to consume the service provided via the package + 1. ingest data to the application + 1. assert the ingested data + +1. clean up everything + +## Publish Crossplane packages + +Crossplane packages are built leveraging `ytt`, for dealing with templating, and the `up` CLI from Upbound, for building the packages and pushing them to an OCI registry (GitHub Packages). + +## Test Crossplane packages + +The testing phase reflects what is done on Carvel's side, which means: + +1. create a [kind cluster](https://kind.sigs.k8s.io/) +1. install [Upbound's Universal Crossplane](https://github.com/upbound/universal-crossplane) +1. run a test script that adheres to a specific naming convention (`scripts/crossplane-e2e-$PACKAGE_PROVIDER-$PACKAGE_NAME.sh`), in order to: + + 1. install the Crossplane provider(s) needed by the package + 1. install the Crossplane package + 1. create a claim in order to trigger the build of the actual managed resources + 1. deploy and test an application + +1. clean up everything + +!!! warning + Testing Crossplane packages in standard GitHub Actions runners might be tricky: + in fact, standard runners are equipped with 2 vCPUs and 7 GB of RAM as of the time of writing (specs [here](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources)) + which might not be enough for serving the amount of APIs that Crossplane and its providers can bring (especially the AWS one). + Unfortunately, this leads to race conditions that might make the workflows either fail or succeed at unpredictable stages. + + However, it is possible to use the very same scripts provided and test the packages on a larger Kubernetes cluster + or make use of **paid** [GitHub-hosted large runners](https://docs.github.com/en/actions/using-github-hosted-runners/using-larger-runners). + Although standard Linux runners are free for public repositories, larger ones are not. + You can take a look at the [per-minutes rates page](https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#per-minute-rates), + to get an overview about the price of the various instances. diff --git a/docs/ci/documentation.md b/docs/ci/documentation.md new file mode 100644 index 0000000..0bbe32d --- /dev/null +++ b/docs/ci/documentation.md @@ -0,0 +1,35 @@ +--- +title: Documentation +--- + +The current documentation is written using Markdown syntax and translated into static web pages +with [`mkdocs`](https://www.mkdocs.org/), which are then published to [GitHub Pages](https://pages.github.com/) and directly linked to the source code repository. + +Via a single `mkdocs.yml` file it is possible to fine-tune `mkdocs`, its extensions and the theme of choice ([Material](https://squidfunk.github.io/mkdocs-material/)), thus creating very nice-looking and tidy pages. +The structure of the resulting website is defined in that file, too. + +Depending on the specific git event, different workflows may be triggered +when docs files get added or updated inside the `docs` folder, or the `mkdocs.yml` file is changed. + +Pull requests to the `main` branch trigger the `build-docs.yml` workflow, that does the build to validate the syntax. +In the following diagram, a pull request is created at commit `D`, and following commits on the same branch (`E` and `F`) cause pull request syncronize events. +All commits `D`, `E` and `F` trigger the `build-docs.yml` workflow. + +When the pull request is accepted and the code gets merged to main (commit `G`), +the `publish-docs.yml` workflow builds and deploys the static website to GitHub Pages. + +```mermaid +%%{init: { 'logLevel': 'debug', 'theme': 'base', 'gitGraph': { 'showBranches': true,'showCommitLabel': true, 'rotateCommitLabel': false } } }%% +gitGraph + commit id: "A" + commit id: "B" + branch docs + checkout docs + commit id: "C" + commit id: "D" type: HIGHLIGHT + commit id: "E" type: HIGHLIGHT + commit id: "F" type: HIGHLIGHT + checkout main + merge docs id: "G" + commit id: "H" +``` diff --git a/docs/ci/index.md b/docs/ci/index.md new file mode 100644 index 0000000..ae19b8e --- /dev/null +++ b/docs/ci/index.md @@ -0,0 +1,31 @@ +--- +title: Continuous Integration +--- + +Continuous integration practices help reduce time-consuming activities as well as human errors, +by defining some standards and automating operations, such as builds and tests. + +In order to maintain a clean and working main branch that can be a trusted source of code and documentation, +any change to its content should be made only via pull requests (PR). +In fact, the PR-related events can trigger workflows to run some repetitive tasks that can be automated, i.e. tests, +thus reducing the toil on code maintainers. + +This repository features a few GitHub Actions workflows to handle different components, such as: + +- [Documentation](./documentation.md) +- [Carvel and Crossplane packages](./carvel-crossplane-packages.md) +- Carvel repository + +!!! note + [GitHub Actions reusable workflows](https://docs.github.com/en/actions/using-workflows/reusing-workflows) are being used in order to simplify + writing and managing pipelines, that become more readable and easier to maintain, as well as make it possible to effectively reuse them + in different pipelines to respond to different events. + + They allow to write specific workflows like functions in programming languages, that can read inputs and produce outputs + that can be passed along to other workflows. + It's important to highlight two things that might not be trivial: + + 1. Environment variables are not transferred from the parent workflow to its children. + In order to pass values from one to the others, inputs/outputs shall be used. + 1. Secrets are not immediately visible in children workflows: + they must either be defined one by one or declare explicit inheritance. More info [here](https://docs.github.com/en/actions/using-workflows/reusing-workflows#passing-secrets-to-nested-workflows). diff --git a/mkdocs.yml b/mkdocs.yml index 24450f0..e77d4b1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -87,7 +87,7 @@ markdown_extensions: custom_fences: - name: mermaid class: mermaid - format: !!python/name:pymdownx.superfences.fence_code_format + format: !!python/name:pymdownx.superfences.fence_code_format - pymdownx.tabbed: alternate_style: true - pymdownx.tasklist: @@ -128,3 +128,7 @@ nav: - crossplane/providers/terraform.md - crossplane/providers/helm.md - crossplane/providers/kubernetes.md +- CI: + - ci/index.md + - ci/documentation.md + - ci/carvel-crossplane-packages.md From 3a06f89a126187c6cbc8bded8a021b216a0e848e Mon Sep 17 00:00:00 2001 From: Joost van der Griendt Date: Fri, 3 Feb 2023 17:18:35 +0100 Subject: [PATCH 35/41] review workflow docs with grammarly --- docs/ci/carvel-crossplane-packages.md | 62 +++++++++++++-------------- docs/ci/documentation.md | 14 +++--- docs/ci/index.md | 22 +++++----- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/docs/ci/carvel-crossplane-packages.md b/docs/ci/carvel-crossplane-packages.md index 4358fce..2747a60 100644 --- a/docs/ci/carvel-crossplane-packages.md +++ b/docs/ci/carvel-crossplane-packages.md @@ -2,7 +2,7 @@ title: Carvel and Crossplane packages --- -The code and configuration for the reference packages must be stored in the `packages` folder, +We store the code and configuration for the reference packages in the `packages` folder, in a structure like the following: ```text @@ -15,11 +15,11 @@ packages ┆ ┆ ``` -- `PROVIDER`: the provider the package is built for (i.e. `aws`, `azure`, `gcp`, `multicloud`). -- `PACKAGING`: the packaging system used (currently it must be either `carvel` or `crossplane`). +- `PROVIDER`: the provider we build the package for (i.e., `aws`, `azure`, `gcp`, `multicloud`). +- `PACKAGING`: the packaging system used (currently, it must be either `carvel` or `crossplane`). - `NAME`: the package name. -Multiple packages can be built and tested in parallel via the `publish-packages.yml` workflow, which is represented in the following diagram. +The following diagram shows that multiple packages can be built and tested in parallel via the `publish-packages.yml` workflow. ```mermaid graph LR @@ -45,38 +45,38 @@ graph LR click G "#test-crossplane-packages" ``` -This workflow runs upon push to either `main` or `develop` branches as well as pull request events targeting the `main` branch. -Only the files matching the `packages/*/*/*/**` glob pattern can actually trigger the workflow: this reflects the packages structure described before -and prevents changes to files outside actual packages directories to trigger it. +This workflow runs upon a push event to either `main` or `develop` branches and pull request events targeting the `main` branch. +Only the files matching the `packages/*/*/*/**` glob pattern trigger the workflow: this reflects the packages structure described before +and prevents changes to files outside actual package directories from triggering it. ## List packages -This job evaluates the differences since the previous commit and builds two lists of packages, for Carvel and Crossplane, to feed the following GitHub Actions matrix jobs with. +This job evaluates the differences since the previous commit and builds two lists of packages, for Carvel and Crossplane, to feed the following GitHub Actions matrix jobs. ## Bump version -The repository is tagged with a new version, bumping the latest one by either the major, minor or patch fields. -Which field is increased depends on the git comment having any of the substrings `#major`, `#minor`, `#patch` (default is set to `patch`). -More info [here](https://github.com/anothrNick/github-tag-action/tree/1.57.0). +The repository is tagged with a new version, bumping the latest one by either the major, minor, or patch fields. +The git comment determines which section of the version we increment. Depending on the occurrence of `#major,` `#minor`, `#patch` (default is `patch`). +You can find more information [here](https://github.com/anothrNick/github-tag-action/tree/1.57.0). ## Publish Carvel packages -The Carvel packages that have been changed from the previous commit are built in parallel and published to GitHub Packages using [Carvel's `kctrl`](https://carvel.dev/kapp-controller/docs/v0.43.2/kctrl-package-authoring/). -`kctrl` also produces packages metadata files that are needed to build Carvel's repository, which are then stored in a separate git branch for each package. +We build the changed Carvel packages in parallel and publish them to GitHub Packages using [Carvel's `kctrl`](https://carvel.dev/kapp-controller/docs/v0.43.2/kctrl-package-authoring/) CLI. +`kctrl` also produces package metadata files needed to build the Carvel repository. We store these metadata files in a separate (temporary) git branch for each package. ## Publish Carvel repository -The packages' metadata files produced in the previous job are collected into one single branch, where `kctrl` is used to author the actual Carvel repository publish it to GitHub Packages in OCI format. -The artefact produced by `kctrl` is a manifest file for the `PackageRepository` resource, useful for quickly deploying the repository on top of a kapp-enabled Kubernetes cluster. -This file is attached to a pre-release, which will then be used in the following job for testing the packages. +We collect the packages' metadata files produced in the previous job into one branch. Finally, we use `kctrl` to author the pre-release Carvel repository and publish it to GitHub Packages in OCI format. +The artifact produced by `kctrl` is a manifest file for the `PackageRepository` resource, useful for quickly deploying the repository on top of a kapp-enabled Kubernetes cluster. +This file is attached to a pre-release; we use this file in the following job for testing the packages. -!!! note "" - This Carvel repository is built for the sole purpose of testing the packages, thus a pre-release is created. - The actual repository release is taken care of by a different workflow. +!!! Note "Carvel Repository" + Because we build this Carvel repository to test the packages, we create a pre-release. + The final repository release is taken care of by a different workflow. ## Test Carvel packages -This phase aims at testing the Carvel packages, by spawning one job per package and running them in parallel on separate GitHub Actions runners. +This phase aims at testing the Carvel packages by spawning one (GitHub Workflow) job per package and running them in parallel on separate GitHub Actions runners. Each job performs the following steps: 1. [install the Carvel suite](https://carvel.dev/#install) @@ -84,43 +84,43 @@ Each job performs the following steps: 1. [install kapp-controller](https://carvel.dev/kapp-controller/docs/v0.43.2/install/) 1. [install secretgen-controller](https://github.com/carvel-dev/secretgen-controller/blob/develop/docs/install.md) 1. install the Carvel repository published in the previous job -1. run a test script that adheres to a specific naming convention (`scripts/carvel-e2e-$PACKAGE_PROVIDER-$PACKAGE_NAME.sh`), in order to: +1. run a test script that adheres to a specific naming convention (`scripts/carvel-e2e-$PACKAGE_PROVIDER-$PACKAGE_NAME.sh`), to: - 1. install the cloud operator specific for the current package (i.e. [ACK](https://aws-controllers-k8s.github.io/community/docs/community/overview/), [ASO](https://azure.github.io/azure-service-operator/), ...) + 1. install the cloud operator specific to the current package (i.e., [ACK](https://aws-controllers-k8s.github.io/community/docs/community/overview/), [ASO](https://azure.github.io/azure-service-operator/), ...) 1. deploy the package with sample input data (need to consider also firewall rules to allow network traffic) 1. deploy an application to consume the service provided via the package - 1. ingest data to the application + 1. ingest data into the application 1. assert the ingested data 1. clean up everything ## Publish Crossplane packages -Crossplane packages are built leveraging `ytt`, for dealing with templating, and the `up` CLI from Upbound, for building the packages and pushing them to an OCI registry (GitHub Packages). +We build Crossplane packages leveraging `ytt`, for dealing with templating, and the `up` CLI from Upbound, for building the packages and pushing them to an OCI registry (GitHub Packages). ## Test Crossplane packages -The testing phase reflects what is done on Carvel's side, which means: +The test phase is similar to Carvel's, which means: 1. create a [kind cluster](https://kind.sigs.k8s.io/) 1. install [Upbound's Universal Crossplane](https://github.com/upbound/universal-crossplane) -1. run a test script that adheres to a specific naming convention (`scripts/crossplane-e2e-$PACKAGE_PROVIDER-$PACKAGE_NAME.sh`), in order to: +1. run a test script that adheres to a specific naming convention (`scripts/crossplane-e2e-$PACKAGE_PROVIDER-$PACKAGE_NAME.sh`), to: 1. install the Crossplane provider(s) needed by the package 1. install the Crossplane package - 1. create a claim in order to trigger the build of the actual managed resources + 1. create a claim to trigger the build of the final managed resources 1. deploy and test an application 1. clean up everything !!! warning Testing Crossplane packages in standard GitHub Actions runners might be tricky: - in fact, standard runners are equipped with 2 vCPUs and 7 GB of RAM as of the time of writing (specs [here](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources)) - which might not be enough for serving the amount of APIs that Crossplane and its providers can bring (especially the AWS one). + In fact, standard runners have two vCPUs and 7 GB of RAM at the time of writing (specs [here](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources)). + This is on the low side for [serving the number of APIs](https://blog.upbound.io/scaling-kubernetes-to-thousands-of-crds/) that Crossplane and its providers can bring (especially the AWS one). Unfortunately, this leads to race conditions that might make the workflows either fail or succeed at unpredictable stages. However, it is possible to use the very same scripts provided and test the packages on a larger Kubernetes cluster or make use of **paid** [GitHub-hosted large runners](https://docs.github.com/en/actions/using-github-hosted-runners/using-larger-runners). Although standard Linux runners are free for public repositories, larger ones are not. - You can take a look at the [per-minutes rates page](https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#per-minute-rates), - to get an overview about the price of the various instances. + However, you can look at the [per-minutes rates page](https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#per-minute-rates), + to get an overview of the price of the various instances. diff --git a/docs/ci/documentation.md b/docs/ci/documentation.md index 0bbe32d..dcbde0a 100644 --- a/docs/ci/documentation.md +++ b/docs/ci/documentation.md @@ -5,18 +5,18 @@ title: Documentation The current documentation is written using Markdown syntax and translated into static web pages with [`mkdocs`](https://www.mkdocs.org/), which are then published to [GitHub Pages](https://pages.github.com/) and directly linked to the source code repository. -Via a single `mkdocs.yml` file it is possible to fine-tune `mkdocs`, its extensions and the theme of choice ([Material](https://squidfunk.github.io/mkdocs-material/)), thus creating very nice-looking and tidy pages. +Via a single `mkdocs.yml` file, it is possible to fine-tune `mkdocs`, its extensions, and the theme of choice ([Material](https://squidfunk.github.io/mkdocs-material/)), thus creating very nice-looking and tidy pages. The structure of the resulting website is defined in that file, too. Depending on the specific git event, different workflows may be triggered -when docs files get added or updated inside the `docs` folder, or the `mkdocs.yml` file is changed. +when docs files get added or updated inside the `docs` folder or the `mkdocs.yml` file is changed. -Pull requests to the `main` branch trigger the `build-docs.yml` workflow, that does the build to validate the syntax. -In the following diagram, a pull request is created at commit `D`, and following commits on the same branch (`E` and `F`) cause pull request syncronize events. -All commits `D`, `E` and `F` trigger the `build-docs.yml` workflow. +Pull requests to the `main` branch trigger the `build-docs.yml` workflow that does the build to validate the syntax. +In the following diagram, we create a pull request at commit `D`, and the following commits on the same branch (`E` and `F`) cause pull request synchronize events. +All commits `D`, `E`, and `F` trigger the `build-docs.yml` workflow. -When the pull request is accepted and the code gets merged to main (commit `G`), -the `publish-docs.yml` workflow builds and deploys the static website to GitHub Pages. +When we accept the pull request, the code gets merged to the main branch (commit `G`), +and the `publish-docs.yml` workflow builds and deploys the static website to GitHub Pages. ```mermaid %%{init: { 'logLevel': 'debug', 'theme': 'base', 'gitGraph': { 'showBranches': true,'showCommitLabel': true, 'rotateCommitLabel': false } } }%% diff --git a/docs/ci/index.md b/docs/ci/index.md index ae19b8e..3a3e429 100644 --- a/docs/ci/index.md +++ b/docs/ci/index.md @@ -5,9 +5,9 @@ title: Continuous Integration Continuous integration practices help reduce time-consuming activities as well as human errors, by defining some standards and automating operations, such as builds and tests. -In order to maintain a clean and working main branch that can be a trusted source of code and documentation, -any change to its content should be made only via pull requests (PR). -In fact, the PR-related events can trigger workflows to run some repetitive tasks that can be automated, i.e. tests, +To maintain a clean and working main branch that can be a trusted source of code and documentation, we make changes exclusively via a pull request(PR). + +The PR-related events trigger workflows to run repetitive tasks that are automated, such as tests, thus reducing the toil on code maintainers. This repository features a few GitHub Actions workflows to handle different components, such as: @@ -18,14 +18,14 @@ This repository features a few GitHub Actions workflows to handle different comp !!! note [GitHub Actions reusable workflows](https://docs.github.com/en/actions/using-workflows/reusing-workflows) are being used in order to simplify - writing and managing pipelines, that become more readable and easier to maintain, as well as make it possible to effectively reuse them - in different pipelines to respond to different events. + writing and managing pipelines that become more readable and easier to maintain, as well as making it possible to reuse them effectively + in other pipelines to respond to different events. - They allow to write specific workflows like functions in programming languages, that can read inputs and produce outputs - that can be passed along to other workflows. + They allow writing specific workflows, like functions in programming languages. These workflows can read inputs and produce outputs + that they pass along to other workflows. It's important to highlight two things that might not be trivial: - 1. Environment variables are not transferred from the parent workflow to its children. - In order to pass values from one to the others, inputs/outputs shall be used. - 1. Secrets are not immediately visible in children workflows: - they must either be defined one by one or declare explicit inheritance. More info [here](https://docs.github.com/en/actions/using-workflows/reusing-workflows#passing-secrets-to-nested-workflows). + 1. A workflow does not automatically transfer Environment variables to its children. + To pass values from one to the others, you use the inputs/outputs. + 1. Secrets are not immediately visible in children's workflows: + They must either be defined one by one or declare explicit inheritance. More info [here](https://docs.github.com/en/actions/using-workflows/reusing-workflows#passing-secrets-to-nested-workflows). From 9c0395868e9a696a75b6bd3dbb46f52002a2a1c0 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Mon, 6 Feb 2023 10:42:52 +0100 Subject: [PATCH 36/41] fix terraform provider's resources api group --- scripts/crossplane-install-tf-provider.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/crossplane-install-tf-provider.sh b/scripts/crossplane-install-tf-provider.sh index 72cdcc4..7279c21 100755 --- a/scripts/crossplane-install-tf-provider.sh +++ b/scripts/crossplane-install-tf-provider.sh @@ -5,4 +5,4 @@ PROVIDER_NAME="upbound-provider-terraform" up controlplane provider install xpkg.upbound.io/upbound/provider-terraform:v0.2.0 --name ${PROVIDER_NAME} || true kubectl wait --for=condition="Healthy" providers.pkg.crossplane.io ${PROVIDER_NAME} -kubectl wait --for=condition=Available=True apiservices.apiregistration.k8s.io v1alpha1.tf.crossplane.io +kubectl wait --for=condition=Available=True apiservices.apiregistration.k8s.io v1beta1.tf.upbound.io From ffe3bc78da8fe792bdeb3107c2b1b19f94a7a9a0 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Mon, 6 Feb 2023 11:06:44 +0100 Subject: [PATCH 37/41] set test app name with overlay --- scripts/crossplane-e2e-multicloud-psql.sh | 4 +-- .../app-overlay.ytt.yml | 28 ++++++++++++++++++ .../crossplane-e2e-multicloud-psql/test.sh | 29 ++++++++++++------- 3 files changed, 49 insertions(+), 12 deletions(-) create mode 100644 scripts/crossplane-e2e-multicloud-psql/app-overlay.ytt.yml diff --git a/scripts/crossplane-e2e-multicloud-psql.sh b/scripts/crossplane-e2e-multicloud-psql.sh index a9a7768..e5ebddf 100755 --- a/scripts/crossplane-e2e-multicloud-psql.sh +++ b/scripts/crossplane-e2e-multicloud-psql.sh @@ -52,7 +52,7 @@ echo "> Installing required providers" ./${SCRIPT_FOLDER}/install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} ./${SCRIPT_FOLDER}/claim-helm-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} -./${SCRIPT_FOLDER}/test.sh ${TEST_APP_NAME} +./${SCRIPT_FOLDER}/test.sh # ./${SCRIPT_FOLDER}/cleanup.sh @@ -66,7 +66,7 @@ sleep 5 ./${SCRIPT_FOLDER}/install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} ./${SCRIPT_FOLDER}/claim-azure-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} -./${SCRIPT_FOLDER}/test.sh ${TEST_APP_NAME} +./${SCRIPT_FOLDER}/test.sh ./crossplane-e2e-multicloud-psql/cleanup.sh diff --git a/scripts/crossplane-e2e-multicloud-psql/app-overlay.ytt.yml b/scripts/crossplane-e2e-multicloud-psql/app-overlay.ytt.yml new file mode 100644 index 0000000..dcb05d8 --- /dev/null +++ b/scripts/crossplane-e2e-multicloud-psql/app-overlay.ytt.yml @@ -0,0 +1,28 @@ +#@ load("@ytt:overlay", "overlay") +#@ load("@ytt:data", "data") + +#@ name = data.values.name + +#@overlay/match by=overlay.all, expects="1+" +--- +metadata: + name: #@ name + labels: + app.kubernetes.io/name: #@ name +spec: + selector: + matchLabels: + app.kubernetes.io/name: #@ name + template: + metadata: + labels: + app.kubernetes.io/name: #@ name + spec: + volumes: + #@overlay/match by="name" + - name: secret-volume + projected: + sources: + #@overlay/match by=overlay.all, expects="1+" + - secret: + name: #@ data.values.secret diff --git a/scripts/crossplane-e2e-multicloud-psql/test.sh b/scripts/crossplane-e2e-multicloud-psql/test.sh index 6f31f36..c359824 100755 --- a/scripts/crossplane-e2e-multicloud-psql/test.sh +++ b/scripts/crossplane-e2e-multicloud-psql/test.sh @@ -1,12 +1,16 @@ #!/usr/bin/env bash -TEST_APP_NAME=$1 +set -euo pipefail + +pushd $(dirname $0) + +TEST_APP_NAME=${TEST_APP_NAME:-"spring-boot-mongo"} echo ">> Installing Test Application" -kubectl apply -f https://raw.githubusercontent.com/joostvdg/spring-boot-postgres/main/kubernetes/deployment.yaml +MANIFEST="https://raw.githubusercontent.com/joostvdg/spring-boot-postgres/main/kubernetes/deployment.yaml" +curl -sSfL ${MANIFEST} | ytt -f - -f app-overlay.ytt.yml -v name=${TEST_APP_NAME} -v secret=${CLAIM_NAME} | kubectl apply -n default -f - kubectl get deployment - echo ">> Waiting on Test Application: ${TEST_APP_NAME}" kubectl get pod kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=${TEST_APP_NAME} --timeout=300s @@ -21,13 +25,18 @@ PORT_FORWARD_PID=$! sleep 10 echo ">> Testing Application" -curl -s "http://localhost:8080" -curl --header "Content-Type: application/json" --request POST --data '{"name":"Piet"}' http://localhost:8080/create -curl --header "Content-Type: application/json" --request POST --data '{"name":"Andrea"}' http://localhost:8080/create -HTTP_RESULT=$(curl -s "http://localhost:8080") -[ $(jq 'map(select(.name =="Piet")) | length'<<<$HTTP_RESULT) -eq 1 ] -[ $(jq 'map(select(.name =="Andrea")) | length'<<<$HTTP_RESULT) -eq 1 ] +curl -sSfL "http://localhost:8080" && echo +echo -n ">> Writing record: " && curl -sSfL --header "Content-Type: application/json" --request POST --data '{"name":"Piet"}' http://localhost:8080/create && echo +echo -n ">> Writing record: " && curl -sSfL --header "Content-Type: application/json" --request POST --data '{"name":"Andrea"}' http://localhost:8080/create && echo + +HTTP_RESULT=$(curl -sSfL "http://localhost:8080") +[ $(jq 'map(select(.name == "Piet")) | length'<<<$HTTP_RESULT) -eq 1 ] +[ $(jq 'map(select(.name == "Andrea")) | length'<<<$HTTP_RESULT) -eq 1 ] + +echo "TEST PASSED" echo ">> Killing Port Forward" echo " > PORT_FORWARD_PID=$PORT_FORWARD_PID" -kill -9 $PORT_FORWARD_PID \ No newline at end of file +kill -9 $PORT_FORWARD_PID + +popd From 462772cd7e38d677905723201c66f5a7992213f7 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Mon, 6 Feb 2023 11:37:41 +0100 Subject: [PATCH 38/41] fix cleanup, extend timeouts --- .github/workflows/reusable-crossplane-test.yml | 2 +- scripts/crossplane-e2e-azure-mongodb.sh | 2 +- scripts/crossplane-e2e-azure-mongodb/claim-instance.sh | 2 +- scripts/crossplane-e2e-multicloud-psql.sh | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/reusable-crossplane-test.yml b/.github/workflows/reusable-crossplane-test.yml index 27bad9a..a837392 100644 --- a/.github/workflows/reusable-crossplane-test.yml +++ b/.github/workflows/reusable-crossplane-test.yml @@ -104,7 +104,7 @@ jobs: run: | if [ -x ${SCRIPT} ]; then ${SCRIPT}; fi env: - SCRIPT: ./scripts/crossplane-e2e-${{ inputs.package_name }}/cleanup.sh + SCRIPT: ./scripts/crossplane-e2e-${{ inputs.package_provider }}-${{ inputs.package_name }}/cleanup.sh CLAIM_NAME: ${{ env.UNIQUE_NAME }} TEST_APP_NAME: ${{ env.UNIQUE_NAME }} CONFIG_NAME: ${{ env.UNIQUE_NAME }} diff --git a/scripts/crossplane-e2e-azure-mongodb.sh b/scripts/crossplane-e2e-azure-mongodb.sh index f4e6ede..951430d 100755 --- a/scripts/crossplane-e2e-azure-mongodb.sh +++ b/scripts/crossplane-e2e-azure-mongodb.sh @@ -2,7 +2,7 @@ set -euo pipefail -trap "echo '###ERROR###' ; !! ; top -b -1 -n 1" ERR +trap "echo '###ERROR###' ; echo !! ; top -b -1 -n 1" ERR echo ">> Local Test" diff --git a/scripts/crossplane-e2e-azure-mongodb/claim-instance.sh b/scripts/crossplane-e2e-azure-mongodb/claim-instance.sh index 1d40720..b781ec5 100755 --- a/scripts/crossplane-e2e-azure-mongodb/claim-instance.sh +++ b/scripts/crossplane-e2e-azure-mongodb/claim-instance.sh @@ -42,7 +42,7 @@ kubectl get secret -n ${CROSSPLANE_NAMESPACE} kubectl get secret echo ">> Waiting for Managed Resources To Get Ready" -kubectl wait --for=condition=ready mongodbinstances.azure.ref.services.apps.tanzu.vmware.com ${CLAIM_NAME} --timeout=400s +kubectl wait --for=condition=ready mongodbinstances.azure.ref.services.apps.tanzu.vmware.com ${CLAIM_NAME} --timeout=10m echo ">> Showing Secrets (2)" kubectl get secret -n ${CROSSPLANE_NAMESPACE} diff --git a/scripts/crossplane-e2e-multicloud-psql.sh b/scripts/crossplane-e2e-multicloud-psql.sh index e5ebddf..c9e84cb 100755 --- a/scripts/crossplane-e2e-multicloud-psql.sh +++ b/scripts/crossplane-e2e-multicloud-psql.sh @@ -2,7 +2,7 @@ set -euo pipefail -trap "echo '###ERROR###' ; !! ; top -b -1 -n 1" ERR +trap "echo '###ERROR###' ; echo !! ; top -b -1 -n 1" ERR echo ">> Local Test" @@ -68,6 +68,6 @@ sleep 5 ./${SCRIPT_FOLDER}/claim-azure-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} ./${SCRIPT_FOLDER}/test.sh -./crossplane-e2e-multicloud-psql/cleanup.sh +./${SCRIPT_FOLDER}/cleanup.sh popd From e086a2fa9e73b2c9e2c1db114194e07135a55094 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Mon, 6 Feb 2023 16:25:04 +0100 Subject: [PATCH 39/41] fix multicloud psql scripts and secret --- .../workflows/reusable-crossplane-test.yml | 3 ++- .../psql/ytt/aws-composition.lib.yml | 9 ++++++- .../psql/ytt/helm-composition.ytt.yml | 6 ++--- .../crossplane/psql/ytt/shared.lib.yml | 1 + scripts/crossplane-e2e-azure-mongodb.sh | 2 +- scripts/crossplane-e2e-multicloud-psql.sh | 26 +++++++++---------- .../claim-helm-instance.sh | 17 +++++++----- .../install-package.sh | 4 ++- .../crossplane-e2e-multicloud-psql/test.sh | 5 +++- scripts/crossplane-install-k8s-provider.sh | 2 +- 10 files changed, 47 insertions(+), 28 deletions(-) diff --git a/.github/workflows/reusable-crossplane-test.yml b/.github/workflows/reusable-crossplane-test.yml index a837392..77565ad 100644 --- a/.github/workflows/reusable-crossplane-test.yml +++ b/.github/workflows/reusable-crossplane-test.yml @@ -39,7 +39,8 @@ jobs: CROSSPLANE_NAMESPACE: upbound-system CONFIG_VERSION: ${{ inputs.package_version }} CONFIG_IMAGE: ${{ inputs.package_registry }}/${{ inputs.package_repository }} - UNIQUE_NAME: ${{ inputs.package_name }}-github-${{ github.run_id }}-${{ github.run_attempt }} + UNIQUE_NAME: gh-${{ github.run_id }}-${{ github.run_attempt }} + STORAGE_CLASS: standard # default storage class for kind steps: diff --git a/packages/multicloud/crossplane/psql/ytt/aws-composition.lib.yml b/packages/multicloud/crossplane/psql/ytt/aws-composition.lib.yml index bba188b..fe5b77b 100644 --- a/packages/multicloud/crossplane/psql/ytt/aws-composition.lib.yml +++ b/packages/multicloud/crossplane/psql/ytt/aws-composition.lib.yml @@ -2,7 +2,7 @@ #@ end - +--- #@ def rdsInstance(crossplaneNamespace, providerConfigRef, publiclyAccessible): name: rdsinstance base: @@ -84,6 +84,7 @@ connectionDetails: - name: securitygroup #@ end +--- #@ def securityGroup(): name: securitygroup base: @@ -109,6 +110,7 @@ patches: toFieldPath: spec.forProvider.region #@ end +--- #@ def securityGroupRule(): name: securitygrouprule base: @@ -131,6 +133,7 @@ patches: toFieldPath: spec.forProvider.region #@ end +--- #@ def subnetGroup(): name: subnetgroup base: @@ -156,6 +159,7 @@ patches: toFieldPath: spec.forProvider.region #@ end +--- #@ def subnet(nameSuffix, nameFormat, availabilityZoneFormat, cidrBlockField): name: #@ "subnet" + nameSuffix base: @@ -197,6 +201,7 @@ patches: fmt: #@ availabilityZoneFormat #@ end +--- #@ def routeTable(): name: routetable base: @@ -225,6 +230,7 @@ patches: fmt: 'rds-%s' #@ end +--- #@ def routeTableAssociation(nameSuffix, subnetNameFormat): name: #@ "routeTableAssociation" + nameSuffix base: @@ -256,6 +262,7 @@ patches: fmt: #@ subnetNameFormat #@ end +--- #@ def route(routeFormat): name: route base: diff --git a/packages/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml b/packages/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml index 470bc47..24085ef 100644 --- a/packages/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml +++ b/packages/multicloud/crossplane/psql/ytt/helm-composition.ytt.yml @@ -51,7 +51,7 @@ spec: fromFieldPath: spec.forProvider.values.global.postgresql.auth.username type: FromFieldPath - name: host - fromFieldPath: metadata.labels.address + fromFieldPath: metadata.annotations.address type: FromFieldPath - name: port value: "5432" @@ -98,7 +98,7 @@ spec: toFieldPath: spec.forProvider.values.global.postgresql.auth.database - type: FromCompositeFieldPath fromFieldPath: status.address - toFieldPath: metadata.labels.address + toFieldPath: metadata.annotations.address - type: ToCompositeFieldPath toFieldPath: status.location fromFieldPath: spec.forProvider.namespace @@ -109,7 +109,7 @@ spec: combine: strategy: string string: - fmt: '%s.%s.svc.cluster.local' + fmt: '%s-postgresql.%s.svc.cluster.local' variables: - fromFieldPath: metadata.name - fromFieldPath: spec.forProvider.namespace diff --git a/packages/multicloud/crossplane/psql/ytt/shared.lib.yml b/packages/multicloud/crossplane/psql/ytt/shared.lib.yml index 7a05beb..1b89072 100644 --- a/packages/multicloud/crossplane/psql/ytt/shared.lib.yml +++ b/packages/multicloud/crossplane/psql/ytt/shared.lib.yml @@ -14,6 +14,7 @@ base: labels: services.apps.tanzu.vmware.com/class: multicloud-psql services.apps.tanzu.vmware.com/infra: #@ infra + type: connection.crossplane.io/v1alpha1 patches: - type: FromCompositeFieldPath fromFieldPath: metadata.labels[crossplane.io/claim-name] diff --git a/scripts/crossplane-e2e-azure-mongodb.sh b/scripts/crossplane-e2e-azure-mongodb.sh index 951430d..81a8adf 100755 --- a/scripts/crossplane-e2e-azure-mongodb.sh +++ b/scripts/crossplane-e2e-azure-mongodb.sh @@ -2,7 +2,7 @@ set -euo pipefail -trap "echo '###ERROR###' ; echo !! ; top -b -1 -n 1" ERR +trap "echo '###ERROR###' ; top -b -1 -n 1" ERR echo ">> Local Test" diff --git a/scripts/crossplane-e2e-multicloud-psql.sh b/scripts/crossplane-e2e-multicloud-psql.sh index c9e84cb..cf4325e 100755 --- a/scripts/crossplane-e2e-multicloud-psql.sh +++ b/scripts/crossplane-e2e-multicloud-psql.sh @@ -2,7 +2,7 @@ set -euo pipefail -trap "echo '###ERROR###' ; echo !! ; top -b -1 -n 1" ERR +trap "echo '###ERROR###' ; top -b -1 -n 1" ERR echo ">> Local Test" @@ -25,7 +25,7 @@ pushd $(dirname $0) # CONFIG_VERSION=${CONFIG_VERSION:-"0.0.1-rc-1"} # CLAIM_NAME=${CLAIM_NAME:-"postgresql-0001"} # TEST_APP_NAME=${TEST_APP_NAME:-"spring-boot-postgres"} -STORAGE_CLASS=${STORAGE_CLASS:-"default"} +export STORAGE_CLASS=${STORAGE_CLASS:-"standard"} # kubectl create namespace ${CROSSPLANE_NAMESPACE} || true @@ -51,23 +51,23 @@ echo "> Installing required providers" ) ./${SCRIPT_FOLDER}/install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} -./${SCRIPT_FOLDER}/claim-helm-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} +./${SCRIPT_FOLDER}/claim-helm-instance.sh ./${SCRIPT_FOLDER}/test.sh # ./${SCRIPT_FOLDER}/cleanup.sh -sleep 5 +# sleep 5 -[ -z "${INSTALL_PROVIDER:-}" ] || ( - ./crossplane-install-azure-provider.sh - ./crossplane-install-k8s-provider.sh - ./crossplane-install-tf-provider.sh -) +# [ -z "${INSTALL_PROVIDER:-}" ] || ( +# ./crossplane-install-azure-provider.sh +# ./crossplane-install-k8s-provider.sh +# ./crossplane-install-tf-provider.sh +# ) -./${SCRIPT_FOLDER}/install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} -./${SCRIPT_FOLDER}/claim-azure-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} -./${SCRIPT_FOLDER}/test.sh +# ./${SCRIPT_FOLDER}/install-package.sh ${CONFIG_NAME} ${CONFIG_IMAGE} ${CONFIG_VERSION} +# ./${SCRIPT_FOLDER}/claim-azure-instance.sh ${CLAIM_NAME} ${STORAGE_CLASS} +# ./${SCRIPT_FOLDER}/test.sh -./${SCRIPT_FOLDER}/cleanup.sh +# ./${SCRIPT_FOLDER}/cleanup.sh popd diff --git a/scripts/crossplane-e2e-multicloud-psql/claim-helm-instance.sh b/scripts/crossplane-e2e-multicloud-psql/claim-helm-instance.sh index 2c37a45..36d715e 100755 --- a/scripts/crossplane-e2e-multicloud-psql/claim-helm-instance.sh +++ b/scripts/crossplane-e2e-multicloud-psql/claim-helm-instance.sh @@ -1,11 +1,13 @@ #!/usr/bin/env bash -CLAIM_NAME=$1 -STORAGE_CLASS=$2 -CROSSPLANE_NAMESPACE=${CROSSPLANE_NAMESPACE:-upbound-system} +set -euo pipefail -echo ">> Claiming a PSQl Instance" -cat <> Claiming a PSQL Instance" +kubectl apply -f - <> Showing Secrets (1)" kubectl get secret -n ${CROSSPLANE_NAMESPACE} kubectl get secret +kubectl get managed +trap 'kubectl get managed ; kubectl describe postgresqlinstances.multi.ref.services.apps.tanzu.vmware.com/${CLAIM_NAME} ; kubectl get xpostgresqlinstances -o yaml ; kubectl get secrets ${CLAIM_NAME} -o yaml' ERR + echo ">> Waiting for Managed Resources To Get Ready" -kubectl wait --for=condition=ready postgresqlinstances.multi.ref.services.apps.tanzu.vmware.com/${CLAIM_NAME} --timeout=400s +kubectl wait --for=condition=ready postgresqlinstances.multi.ref.services.apps.tanzu.vmware.com/${CLAIM_NAME} --timeout=120s # We can also wait on the "release" echo ">> Showing Secrets (2)" diff --git a/scripts/crossplane-e2e-multicloud-psql/install-package.sh b/scripts/crossplane-e2e-multicloud-psql/install-package.sh index b9da69b..fe5983a 100755 --- a/scripts/crossplane-e2e-multicloud-psql/install-package.sh +++ b/scripts/crossplane-e2e-multicloud-psql/install-package.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -euo pipefail + CONFIG_NAME=${1:-${CONFIG_NAME:-}} CONFIG_IMAGE=${2:-${CONFIG_IMAGE:-}} CONFIG_VERSION=${3:-${CONFIG_VERSION:-}} @@ -16,7 +18,7 @@ metadata: spec: ignoreCrossplaneConstraints: true package: ${CONFIG_IMAGE}:${CONFIG_VERSION} - packagePullPolicy: Allways + packagePullPolicy: Always revisionActivationPolicy: Automatic revisionHistoryLimit: 3 skipDependencyResolution: true diff --git a/scripts/crossplane-e2e-multicloud-psql/test.sh b/scripts/crossplane-e2e-multicloud-psql/test.sh index c359824..7558287 100755 --- a/scripts/crossplane-e2e-multicloud-psql/test.sh +++ b/scripts/crossplane-e2e-multicloud-psql/test.sh @@ -11,11 +11,14 @@ MANIFEST="https://raw.githubusercontent.com/joostvdg/spring-boot-postgres/main/k curl -sSfL ${MANIFEST} | ytt -f - -f app-overlay.ytt.yml -v name=${TEST_APP_NAME} -v secret=${CLAIM_NAME} | kubectl apply -n default -f - kubectl get deployment +trap 'kubectl get pods -o yaml' ERR + echo ">> Waiting on Test Application: ${TEST_APP_NAME}" kubectl get pod kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=${TEST_APP_NAME} --timeout=300s -kubectl describe pod -l app.kubernetes.io/name=${TEST_APP_NAME} +kubectl describe pod -l app.kubernetes.io/name=${TEST_APP_NAME} + sleep 10 echo ">> Starting Port Forward" diff --git a/scripts/crossplane-install-k8s-provider.sh b/scripts/crossplane-install-k8s-provider.sh index 3cc80bf..0f8bd5a 100755 --- a/scripts/crossplane-install-k8s-provider.sh +++ b/scripts/crossplane-install-k8s-provider.sh @@ -18,5 +18,5 @@ spec: source: InjectedIdentity EOF -SA=$(kubectl -n ${CROSSPLANE_NAMESPACE} get sa -o name | grep provider-helm | sed -e 's|serviceaccount\/|'${CROSSPLANE_NAMESPACE}':|g') +SA=$(kubectl -n ${CROSSPLANE_NAMESPACE} get sa -o name | grep provider-kubernetes | sed -e 's|serviceaccount\/|'${CROSSPLANE_NAMESPACE}':|g') kubectl create clusterrolebinding provider-kubernetes-admin-binding --clusterrole cluster-admin --serviceaccount="${SA}" || true From 2f5e482dcb3cf2cc5885a79f5efa43a3454de642 Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Wed, 8 Feb 2023 14:59:06 +0100 Subject: [PATCH 40/41] fix publish docs permissions --- .github/workflows/publish-docs.yml | 1 + mkdocs.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index f97a190..792d5e2 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -2,6 +2,7 @@ name: Publish docs permissions: pages: write + contents: write on: push: diff --git a/mkdocs.yml b/mkdocs.yml index e77d4b1..3fedf12 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -131,4 +131,4 @@ nav: - CI: - ci/index.md - ci/documentation.md - - ci/carvel-crossplane-packages.md + - ci/carvel-crossplane-packages.md \ No newline at end of file From 322549afcd4810a493c00064738882dd9ac8847a Mon Sep 17 00:00:00 2001 From: Matteo Magni Date: Wed, 8 Feb 2023 15:36:25 +0100 Subject: [PATCH 41/41] fix workflow permissions --- .github/workflows/publish-packages.yml | 1 + .github/workflows/reusable-bump-version.yml | 6 ++++++ .../workflows/reusable-carvel-publish-repo.yml | 15 +++++++++------ .github/workflows/reusable-carvel-test.yml | 2 +- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index cdbdbbb..33c2ff5 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -7,6 +7,7 @@ concurrency: permissions: packages: write contents: write + pull-requests: write on: push: diff --git a/.github/workflows/reusable-bump-version.yml b/.github/workflows/reusable-bump-version.yml index c7529dc..ba869c5 100644 --- a/.github/workflows/reusable-bump-version.yml +++ b/.github/workflows/reusable-bump-version.yml @@ -32,6 +32,12 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - name: Define default branch + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + echo "DEFAULT_BRANCH=${{ github.head_ref }}" >> $GITHUB_ENV + fi + - name: Bump version and push tag id: bump uses: anothrNick/github-tag-action@1.57.0 diff --git a/.github/workflows/reusable-carvel-publish-repo.yml b/.github/workflows/reusable-carvel-publish-repo.yml index 18ff4e7..b7a24e2 100644 --- a/.github/workflows/reusable-carvel-publish-repo.yml +++ b/.github/workflows/reusable-carvel-publish-repo.yml @@ -114,15 +114,18 @@ jobs: - name: Create pull request id: pr - if: inputs.prepare_repo_pr run: | - COMMIT_MESSAGE="$(git log --pretty=format:'%s' HEAD~..HEAD)" + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + echo "PULL_REQUEST_NUMBER=${{ github.event.number }}" >> $GITHUB_OUTPUT + elif [[ "${{ inputs.prepare_repo_pr }}" =~ ^[tT][rR][uU][eE]$ ]]; then + COMMIT_MESSAGE="$(git log --pretty=format:'%s' HEAD~..HEAD)" - # create the pull request against CURRENT_BRANCH - gh pr create --base ${CURRENT_BRANCH} --title "Carvel repository release at commit ${{ github.sha }}" --body "${COMMIT_MESSAGE}" + # create the pull request against CURRENT_BRANCH + gh pr create --base ${CURRENT_BRANCH} --title "Carvel repository release at commit ${{ github.sha }}" --body "${COMMIT_MESSAGE}" - # get pull request number - echo "PULL_REQUEST_NUMBER=$(gh pr view --json number -q '.number' ${REPO_BRANCH_NAME})" >> $GITHUB_OUTPUT + # get pull request number + echo "PULL_REQUEST_NUMBER=$(gh pr view --json number -q '.number' ${REPO_BRANCH_NAME})" >> $GITHUB_OUTPUT + fi env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/reusable-carvel-test.yml b/.github/workflows/reusable-carvel-test.yml index 5608eb5..17b21e5 100644 --- a/.github/workflows/reusable-carvel-test.yml +++ b/.github/workflows/reusable-carvel-test.yml @@ -141,7 +141,7 @@ jobs: if [[ "$PRERELEASE" =~ ^[tT][rR][uU][eE]$ ]]; then if [ -x ${SCRIPT} ]; then ${SCRIPT} - else + elif [ ! -z "$PULL_REQUEST_NUMBER" ]; then # no pre-release test script provided LABEL=untested gh label create -c ffaa00 --force $LABEL