json-response-preprocess

.github/workflows/build.yml

@@ -26,6 +26,49 @@ env:
 
 jobs:
 
+
+  test_python_package_build:
+    # id: test_python_package_build
+    name: Test Python Package Build
+    runs-on: ubuntu-22.04
+    timeout-minutes: ${{ vars.DEFAULT_JOB_TIMEOUT_MIN == '' && 120 || vars.DEFAULT_JOB_TIMEOUT_MIN }}
+    steps:
+
+    - name: Check out code into the Go module directory
+      uses: actions/[email protected]
+      with:
+        repository: ${{ env.STACKQL_CORE_REPOSITORY }}
+        ref: ${{ env.STACKQL_CORE_REF }}
+        token: ${{ secrets.CI_STACKQL_PACKAGE_DOWNLOAD_TOKEN  }}
+        path: stackql-core-pkg
+
+    - name: Setup Python
+      uses: actions/[email protected]
+      with:
+        cache: pip
+        python-version: '3.12' 
+
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        version: 1.8.3
+        virtualenvs-create: true
+        virtualenvs-in-project: false
+        virtualenvs-path: stackql-core-pkg/my-custom-path
+        installer-parallel: true
+
+
+    - name: Build package
+      working-directory: stackql-core-pkg
+      run: |
+        cicd/util/01-build-robot-lib.sh
+
+    - name: Upload python package artifact
+      uses: actions/[email protected]
+      with:
+        name: python-package-dist-folder
+        path: stackql-core-pkg/test/dist
+
   winbuild:
     name: Windows Build
     runs-on: windows-latest
@@ -85,6 +128,7 @@ jobs:
       run: go test -timeout 240s -v ./...
 
   linuxbuild:
+    needs: test_python_package_build
     name: Linux Build
     runs-on: ubuntu-latest
     steps:
@@ -106,7 +150,25 @@ jobs:
       uses: actions/[email protected]
       with:
         # cache: pip # this requires requirements in source control
-        python-version: '3.11' 
+        python-version: '3.12'
+
+    - name: Install python dependencies
+      run: |
+        echo "Installing python dependencies"
+        pip3 install -r cicd/testing-requirements.txt
+
+    - name: Download python package dist folder
+      uses: actions/[email protected]
+      with:
+        name: python-package-dist-folder
+        path: test/dist
+
+    - name: Install python testing package
+      run: |
+        echo "Inspecting python package"
+        for file in test/dist/*.whl; do
+          pip3 install "$file" --force-reinstall
+        done 
 
     - name: Get dependencies
       run: |
@@ -262,6 +324,24 @@ jobs:
           echo "Core Test passed with matching public key algorithm: '$publicKeyAlgorithm'"
         fi
 
+    - name: Prepare for robot test run
+      run: |
+        pgrep -f flask | xargs kill -9 || true
+        python ${{ github.workspace }}/stackql-core/test/python/stackql_test_tooling/registry_rewrite.py --srcdir "${{ github.workspace }}/test/registry/src" --destdir "${{ github.workspace }}/test/registry-mocked/src"
+        openssl req -x509 -keyout ${{ github.workspace }}/test/credentials/pg_server_key.pem  -out ${{ github.workspace }}/test/credentials/pg_server_cert.pem   -config ${{ github.workspace }}/stackql-core/test/server/mtls/openssl.cnf -days 365
+        openssl req -x509 -keyout ${{ github.workspace }}/test/credentials/pg_client_key.pem  -out ${{ github.workspace }}/test/credentials/pg_client_cert.pem   -config ${{ github.workspace }}/stackql-core/test/server/mtls/openssl.cnf -days 365
+        openssl req -x509 -keyout ${{ github.workspace }}/test/credentials/pg_rubbish_key.pem -out ${{ github.workspace }}/test/credentials/pg_rubbish_cert.pem  -config ${{ github.workspace }}/stackql-core/test/server/mtls/openssl.cnf -days 365
+
+    - name: Run mocked robot tests
+      run: |
+        export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/test/python"
+        robot -d test/robot/reports/mocked test/robot/cli/mocked
+
+    - name: Output from mocked robot tests
+      if: always()
+      run: |
+        cat test/robot/reports/mocked/output.xml
+
   macosbuild:
     name: MacOS Build
     runs-on: macos-latest

.gitignore

@@ -2,3 +2,8 @@
 *.prof
 opint
 .DS_Store
+dist/
+.venv/
+__pycache__/
+*.py[co]
+stackql-core/

anysdk/operation_store.go

@@ -1432,7 +1432,14 @@ func (op *standardOpenAPIOperationStore) isOverridable(httpResponse *http.Respon
 	if isExpectedResponse {
 		responseTransform, responseTransformExists := expectedResponse.GetTransform()
 		overrideMediaType := expectedResponse.GetOverrrideBodyMediaType()
-		return responseTransformExists && responseTransform.GetType() == "golang_template_mxj_v0.1.0" && overrideMediaType != ""
+		if !responseTransformExists {
+			return false
+		}
+		streamTransformerFactory := stream_transform.NewStreamTransformerFactory(
+			responseTransform.GetType(),
+			responseTransform.GetBody(),
+		)
+		return overrideMediaType != "" && streamTransformerFactory.IsTransformable()
 	}
 	return false
 }
@@ -1447,19 +1454,24 @@ func (op *standardOpenAPIOperationStore) getOverridenResponse(httpResponse *http
 	if isExpectedResponse {
 		responseTransform, responseTransformExists := expectedResponse.GetTransform()
 		overrideMediaType := expectedResponse.GetOverrrideBodyMediaType()
-		if responseTransformExists && responseTransform.GetType() == "golang_template_mxj_v0.1.0" {
+		if responseTransformExists {
 			input := string(bodyBytes)
-			tmpl := responseTransform.GetBody()
-			inStream := stream_transform.NewXMLBestEffortReader(bytes.NewBufferString(input))
-			outStream := bytes.NewBuffer(nil)
-			tfm, err := stream_transform.NewTemplateStreamTransformer(tmpl, inStream, outStream)
-			if err != nil {
-				return nil, fmt.Errorf("template stream transform error: %v", err)
+			streamTransformerFactory := stream_transform.NewStreamTransformerFactory(
+				responseTransform.GetType(),
+				responseTransform.GetBody(),
+			)
+			if !streamTransformerFactory.IsTransformable() {
+				return nil, fmt.Errorf("unsupported template type: %s", responseTransform.GetType())
 			}
-			if err := tfm.Transform(); err != nil {
+			tfm, err := streamTransformerFactory.GetTransformer(input)
+			if err != nil {
 				return nil, fmt.Errorf("failed to transform: %v", err)
 			}
-			// bodyBytes = outStream.Bytes()
+			tfmErr := tfm.Transform()
+			if tfmErr != nil {
+				return nil, fmt.Errorf("failed to transform: %v", tfmErr)
+			}
+			outStream := tfm.GetOutStream()
 			if overrideMediaType == "" {
 				overrideMediaType = media.MediaTypeJson
 			}

anysdk/operation_store_test.go

@@ -109,6 +109,7 @@ func TestTransformedXMLHTTPHandle(t *testing.T) {
 	assert.Assert(t, ops != nil)
 
 	processed, err := ops.ProcessResponse(res)
+	t.Logf("err: %v", err)
 	assert.NilError(t, err)
 	processedResponse, ok := processed.GetResponse()
 	assert.Assert(t, ok)

cicd/scripts/local/01-gather.sh

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+>&2 echo "requires git version >= 2.45"
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+source "${SCRIPT_DIR}/context.sh"
+
+cd "${REPOSITORY_ROOT_DIR}"
+
+git clone --revision=refs/heads/main  https://github.com/stackql/stackql.git stackql-core
+

cicd/scripts/local/02-build-stackql-test-package.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+>&2 echo "requires all of requirements.txt"
+
+SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+source "${SCRIPT_DIR}/context.sh"
+
+PACKAGE_ROOT="${STACKQL_CORE_DIR}/test"
+
+rm -f ${PACKAGE_ROOT}/dist/*.whl || true
+
+${STACKQL_CORE_DIR}/cicd/util/01-build-robot-lib.sh
+
+filez="$(ls ${PACKAGE_ROOT}/dist/*.whl)" || true
+
+if [ "${filez}" = "" ]; then
+    >&2 echo "No wheel files found in ${PACKAGE_ROOT}/dist. Please check the build process."
+    exit 1
+else
+    echo "Wheel files found in ${PACKAGE_ROOT}/dist: ${filez}"
+fi
+
+
+
+

cicd/scripts/local/03-install-stackql-test-package.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+>&2 echo "requires all of requirements.txt"
+
+SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+source "${SCRIPT_DIR}/context.sh"
+
+if [ ! -d "${REPOSITORY_ROOT_DIR}/.venv" ]; then
+  >&2 echo "No existing virtual environment, creating one..."
+  >&2 echo "Creating virtual environment in ${REPOSITORY_ROOT_DIR}/.venv"
+  python -m venv "${REPOSITORY_ROOT_DIR}/.venv"
+  >&2 echo "Virtual environment created."
+else
+  >&2 echo "Using existing virtual environment in ${REPOSITORY_ROOT_DIR}/.venv"
+fi
+
+source "${REPOSITORY_ROOT_DIR}/.venv/bin/activate"
+
+pip install -r "${REPOSITORY_ROOT_DIR}/cicd/testing-requirements.txt"
+
+PACKAGE_ROOT="${STACKQL_CORE_DIR}/test"
+
+filez="$(ls ${PACKAGE_ROOT}/dist/*.whl)" || true
+
+if [ "${filez}" = "" ]; then
+    >&2 echo "No wheel files found in ${PACKAGE_ROOT}/dist. Please check the build process."
+    exit 1
+else
+    echo "Wheel files found in ${PACKAGE_ROOT}/dist: ${filez}"
+fi
+
+
+for file in ${PACKAGE_ROOT}/dist/*.whl; do
+    pip3 install "$file" --force-reinstall
+done

cicd/scripts/local/04-setup-testing-dependencies.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+>&2 echo "requires all of requirements.txt"
+
+SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+source "${SCRIPT_DIR}/context.sh"
+
+# Check if the virtual environment exists
+if [ ! -d "${STACKQL_CORE_DIR}/.venv" ]; then
+    >&2 echo "Virtual environment not found. Please create it first."
+    exit 1
+fi
+
+source "${STACKQL_CORE_DIR}/.venv/bin/activate"
+
+python ${STACKQL_CORE_DIR}/test/python/stackql_test_tooling/registry_rewrite.py --srcdir "${REPOSITORY_ROOT_DIR}/test/registry/src" --destdir "${REPOSITORY_ROOT_DIR}/test/registry-mocked/src"
+
+openssl req -x509 -keyout ${REPOSITORY_ROOT_DIR}/test/credentials/pg_server_key.pem  -out ${REPOSITORY_ROOT_DIR}/test/credentials/pg_server_cert.pem   -config ${STACKQL_CORE_DIR}/test/server/mtls/openssl.cnf -days 365
+openssl req -x509 -keyout ${REPOSITORY_ROOT_DIR}/test/credentials/pg_client_key.pem  -out ${REPOSITORY_ROOT_DIR}/test/credentials/pg_client_cert.pem   -config ${STACKQL_CORE_DIR}/test/server/mtls/openssl.cnf -days 365
+openssl req -x509 -keyout ${REPOSITORY_ROOT_DIR}/test/credentials/pg_rubbish_key.pem -out ${REPOSITORY_ROOT_DIR}/test/credentials/pg_rubbish_cert.pem  -config ${STACKQL_CORE_DIR}/test/server/mtls/openssl.cnf -days 365

cicd/scripts/local/11-run-mocked.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+>&2 echo "requires all of requirements.txt"
+
+SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+source "${SCRIPT_DIR}/context.sh"
+
+cd "${REPOSITORY_ROOT_DIR}"
+
+source "${REPOSITORY_ROOT_DIR}/.venv/bin/activate"
+
+export PYTHONPATH="${PYTHONPATH}:${REPOSITORY_ROOT_DIR}/test/python"
+
+robot -d test/robot/reports/mocked test/robot/cli/mocked
+

cicd/scripts/local/99-clean.sh

@@ -0,0 +1,13 @@
+#! /usr/bin/env bash
+
+>&2 echo "cleaning dependent repositories from local file system"
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+REPOSITORY_ROOT_DIR="$(realpath ${SCRIPT_DIR}/../../..)"
+
+cd "${REPOSITORY_ROOT_DIR}"
+
+rm -rf stackql-core    || true
+rm -rf stackql-any-sdk || true
+

cicd/scripts/local/context.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+
+SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+export REPOSITORY_ROOT_DIR="$(realpath ${SCRIPT_DIR}/../../..)"
+
+export STACKQL_CORE_DIR="${STACKQL_CORE_DIR:-"${REPOSITORY_ROOT_DIR}/stackql-core}"}"
+
+
+checkPoetry () {
+    if ! command -v poetry &> /dev/null
+    then
+        >&2 echo "Poetry is not installed. Please install it first." 
+        exit 1
+    fi
+}

cicd/scripts/local/local_scripts.md

@@ -0,0 +1,15 @@
+
+
+Here is a typical example, from the root oif this repository, assuming you have the core repository locally at `../stackql-devel`:
+
+```bash
+
+env STACKQL_CORE_DIR="$(realpath ../stackql-devel)" cicd/scripts/local/02-build-stackql-test-package.sh
+
+env STACKQL_CORE_DIR="$(realpath ../stackql-devel)" cicd/scripts/local/03-install-stackql-test-package.sh
+
+env STACKQL_CORE_DIR="$(realpath ../stackql-devel)" cicd/scripts/local/04-setup-testing-dependencies.sh
+
+env GCP_SERVICE_ACCOUNT_KEY="$(cat test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json)" cicd/scripts/local/11-run-mocked.sh
+
+```

cicd/testing-requirements.txt

@@ -0,0 +1,4 @@
+PyYaml==6.0.2
+requests==2.32.3
+robotframework==7.0.1
+tabulate==0.9.0