Skip to content

Commit 210c896

Browse files
general-kroll-4-lifegeneral-kroll-4-life
authored
Merge pull request #35 from stackql/feature/templating-expand-plus
json-response-preprocess
2 parents 6b5397c + eee43d0 commit 210c896

28 files changed

+650
-62
lines changed

‎.github/workflows/build.yml

Lines changed: 81 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,49 @@ env:
2626

2727
jobs:
2828

29+
30+
test_python_package_build:
31+
# id: test_python_package_build
32+
name: Test Python Package Build
33+
runs-on: ubuntu-22.04
34+
timeout-minutes: ${{ vars.DEFAULT_JOB_TIMEOUT_MIN == '' && 120 || vars.DEFAULT_JOB_TIMEOUT_MIN }}
35+
steps:
36+
37+
- name: Check out code into the Go module directory
38+
uses: actions/[email protected]
39+
with:
40+
repository: ${{ env.STACKQL_CORE_REPOSITORY }}
41+
ref: ${{ env.STACKQL_CORE_REF }}
42+
token: ${{ secrets.CI_STACKQL_PACKAGE_DOWNLOAD_TOKEN }}
43+
path: stackql-core-pkg
44+
45+
- name: Setup Python
46+
uses: actions/[email protected]
47+
with:
48+
cache: pip
49+
python-version: '3.12'
50+
51+
- name: Install Poetry
52+
uses: snok/install-poetry@v1
53+
with:
54+
version: 1.8.3
55+
virtualenvs-create: true
56+
virtualenvs-in-project: false
57+
virtualenvs-path: stackql-core-pkg/my-custom-path
58+
installer-parallel: true
59+
60+
61+
- name: Build package
62+
working-directory: stackql-core-pkg
63+
run: |
64+
cicd/util/01-build-robot-lib.sh
65+
66+
- name: Upload python package artifact
67+
uses: actions/[email protected]
68+
with:
69+
name: python-package-dist-folder
70+
path: stackql-core-pkg/test/dist
71+
2972
winbuild:
3073
name: Windows Build
3174
runs-on: windows-latest
@@ -85,6 +128,7 @@ jobs:
85128
run: go test -timeout 240s -v ./...
86129

87130
linuxbuild:
131+
needs: test_python_package_build
88132
name: Linux Build
89133
runs-on: ubuntu-latest
90134
steps:
@@ -106,7 +150,25 @@ jobs:
106150
uses: actions/[email protected]
107151
with:
108152
# cache: pip # this requires requirements in source control
109-
python-version: '3.11'
153+
python-version: '3.12'
154+
155+
- name: Install python dependencies
156+
run: |
157+
echo "Installing python dependencies"
158+
pip3 install -r cicd/testing-requirements.txt
159+
160+
- name: Download python package dist folder
161+
uses: actions/[email protected]
162+
with:
163+
name: python-package-dist-folder
164+
path: test/dist
165+
166+
- name: Install python testing package
167+
run: |
168+
echo "Inspecting python package"
169+
for file in test/dist/*.whl; do
170+
pip3 install "$file" --force-reinstall
171+
done
110172
111173
- name: Get dependencies
112174
run: |
@@ -262,6 +324,24 @@ jobs:
262324
echo "Core Test passed with matching public key algorithm: '$publicKeyAlgorithm'"
263325
fi
264326
327+
- name: Prepare for robot test run
328+
run: |
329+
pgrep -f flask | xargs kill -9 || true
330+
python ${{ github.workspace }}/stackql-core/test/python/stackql_test_tooling/registry_rewrite.py --srcdir "${{ github.workspace }}/test/registry/src" --destdir "${{ github.workspace }}/test/registry-mocked/src"
331+
openssl req -x509 -keyout ${{ github.workspace }}/test/credentials/pg_server_key.pem -out ${{ github.workspace }}/test/credentials/pg_server_cert.pem -config ${{ github.workspace }}/stackql-core/test/server/mtls/openssl.cnf -days 365
332+
openssl req -x509 -keyout ${{ github.workspace }}/test/credentials/pg_client_key.pem -out ${{ github.workspace }}/test/credentials/pg_client_cert.pem -config ${{ github.workspace }}/stackql-core/test/server/mtls/openssl.cnf -days 365
333+
openssl req -x509 -keyout ${{ github.workspace }}/test/credentials/pg_rubbish_key.pem -out ${{ github.workspace }}/test/credentials/pg_rubbish_cert.pem -config ${{ github.workspace }}/stackql-core/test/server/mtls/openssl.cnf -days 365
334+
335+
- name: Run mocked robot tests
336+
run: |
337+
export PYTHONPATH="${PYTHONPATH}:${{ github.workspace }}/test/python"
338+
robot -d test/robot/reports/mocked test/robot/cli/mocked
339+
340+
- name: Output from mocked robot tests
341+
if: always()
342+
run: |
343+
cat test/robot/reports/mocked/output.xml
344+
265345
macosbuild:
266346
name: MacOS Build
267347
runs-on: macos-latest

‎.gitignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,8 @@
22
*.prof
33
opint
44
.DS_Store
5+
dist/
6+
.venv/
7+
__pycache__/
8+
*.py[co]
9+
stackql-core/

‎anysdk/operation_store.go

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1432,7 +1432,14 @@ func (op *standardOpenAPIOperationStore) isOverridable(httpResponse *http.Respon
14321432
if isExpectedResponse {
14331433
responseTransform, responseTransformExists := expectedResponse.GetTransform()
14341434
overrideMediaType := expectedResponse.GetOverrrideBodyMediaType()
1435-
return responseTransformExists && responseTransform.GetType() == "golang_template_mxj_v0.1.0" && overrideMediaType != ""
1435+
if !responseTransformExists {
1436+
return false
1437+
}
1438+
streamTransformerFactory := stream_transform.NewStreamTransformerFactory(
1439+
responseTransform.GetType(),
1440+
responseTransform.GetBody(),
1441+
)
1442+
return overrideMediaType != "" && streamTransformerFactory.IsTransformable()
14361443
}
14371444
return false
14381445
}
@@ -1447,19 +1454,24 @@ func (op *standardOpenAPIOperationStore) getOverridenResponse(httpResponse *http
14471454
if isExpectedResponse {
14481455
responseTransform, responseTransformExists := expectedResponse.GetTransform()
14491456
overrideMediaType := expectedResponse.GetOverrrideBodyMediaType()
1450-
if responseTransformExists && responseTransform.GetType() == "golang_template_mxj_v0.1.0" {
1457+
if responseTransformExists {
14511458
input := string(bodyBytes)
1452-
tmpl := responseTransform.GetBody()
1453-
inStream := stream_transform.NewXMLBestEffortReader(bytes.NewBufferString(input))
1454-
outStream := bytes.NewBuffer(nil)
1455-
tfm, err := stream_transform.NewTemplateStreamTransformer(tmpl, inStream, outStream)
1456-
if err != nil {
1457-
return nil, fmt.Errorf("template stream transform error: %v", err)
1459+
streamTransformerFactory := stream_transform.NewStreamTransformerFactory(
1460+
responseTransform.GetType(),
1461+
responseTransform.GetBody(),
1462+
)
1463+
if !streamTransformerFactory.IsTransformable() {
1464+
return nil, fmt.Errorf("unsupported template type: %s", responseTransform.GetType())
14581465
}
1459-
if err := tfm.Transform(); err != nil {
1466+
tfm, err := streamTransformerFactory.GetTransformer(input)
1467+
if err != nil {
14601468
return nil, fmt.Errorf("failed to transform: %v", err)
14611469
}
1462-
// bodyBytes = outStream.Bytes()
1470+
tfmErr := tfm.Transform()
1471+
if tfmErr != nil {
1472+
return nil, fmt.Errorf("failed to transform: %v", tfmErr)
1473+
}
1474+
outStream := tfm.GetOutStream()
14631475
if overrideMediaType == "" {
14641476
overrideMediaType = media.MediaTypeJson
14651477
}

‎anysdk/operation_store_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ func TestTransformedXMLHTTPHandle(t *testing.T) {
109109
assert.Assert(t, ops != nil)
110110

111111
processed, err := ops.ProcessResponse(res)
112+
t.Logf("err: %v", err)
112113
assert.NilError(t, err)
113114
processedResponse, ok := processed.GetResponse()
114115
assert.Assert(t, ok)

‎cicd/scripts/local/01-gather.sh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#! /usr/bin/env bash
2+
3+
>&2 echo "requires git version >= 2.45"
4+
5+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
6+
7+
source "${SCRIPT_DIR}/context.sh"
8+
9+
cd "${REPOSITORY_ROOT_DIR}"
10+
11+
git clone --revision=refs/heads/main https://github.com/stackql/stackql.git stackql-core
12+

‎cicd/scripts/local/02-build-stackql-test-package.sh

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/usr/bin/env bash
2+
3+
>&2 echo "requires all of requirements.txt"
4+
5+
SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
6+
7+
source "${SCRIPT_DIR}/context.sh"
8+
9+
PACKAGE_ROOT="${STACKQL_CORE_DIR}/test"
10+
11+
rm -f ${PACKAGE_ROOT}/dist/*.whl || true
12+
13+
${STACKQL_CORE_DIR}/cicd/util/01-build-robot-lib.sh
14+
15+
filez="$(ls ${PACKAGE_ROOT}/dist/*.whl)" || true
16+
17+
if [ "${filez}" = "" ]; then
18+
>&2 echo "No wheel files found in ${PACKAGE_ROOT}/dist. Please check the build process."
19+
exit 1
20+
else
21+
echo "Wheel files found in ${PACKAGE_ROOT}/dist: ${filez}"
22+
fi
23+
24+
25+
26+

‎cicd/scripts/local/03-install-stackql-test-package.sh

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/usr/bin/env bash
2+
3+
>&2 echo "requires all of requirements.txt"
4+
5+
SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
6+
7+
source "${SCRIPT_DIR}/context.sh"
8+
9+
if [ ! -d "${REPOSITORY_ROOT_DIR}/.venv" ]; then
10+
>&2 echo "No existing virtual environment, creating one..."
11+
>&2 echo "Creating virtual environment in ${REPOSITORY_ROOT_DIR}/.venv"
12+
python -m venv "${REPOSITORY_ROOT_DIR}/.venv"
13+
>&2 echo "Virtual environment created."
14+
else
15+
>&2 echo "Using existing virtual environment in ${REPOSITORY_ROOT_DIR}/.venv"
16+
fi
17+
18+
source "${REPOSITORY_ROOT_DIR}/.venv/bin/activate"
19+
20+
pip install -r "${REPOSITORY_ROOT_DIR}/cicd/testing-requirements.txt"
21+
22+
PACKAGE_ROOT="${STACKQL_CORE_DIR}/test"
23+
24+
filez="$(ls ${PACKAGE_ROOT}/dist/*.whl)" || true
25+
26+
if [ "${filez}" = "" ]; then
27+
>&2 echo "No wheel files found in ${PACKAGE_ROOT}/dist. Please check the build process."
28+
exit 1
29+
else
30+
echo "Wheel files found in ${PACKAGE_ROOT}/dist: ${filez}"
31+
fi
32+
33+
34+
for file in ${PACKAGE_ROOT}/dist/*.whl; do
35+
pip3 install "$file" --force-reinstall
36+
done

‎cicd/scripts/local/04-setup-testing-dependencies.sh

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/usr/bin/env bash
2+
3+
>&2 echo "requires all of requirements.txt"
4+
5+
SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
6+
7+
source "${SCRIPT_DIR}/context.sh"
8+
9+
# Check if the virtual environment exists
10+
if [ ! -d "${STACKQL_CORE_DIR}/.venv" ]; then
11+
>&2 echo "Virtual environment not found. Please create it first."
12+
exit 1
13+
fi
14+
15+
source "${STACKQL_CORE_DIR}/.venv/bin/activate"
16+
17+
python ${STACKQL_CORE_DIR}/test/python/stackql_test_tooling/registry_rewrite.py --srcdir "${REPOSITORY_ROOT_DIR}/test/registry/src" --destdir "${REPOSITORY_ROOT_DIR}/test/registry-mocked/src"
18+
19+
openssl req -x509 -keyout ${REPOSITORY_ROOT_DIR}/test/credentials/pg_server_key.pem -out ${REPOSITORY_ROOT_DIR}/test/credentials/pg_server_cert.pem -config ${STACKQL_CORE_DIR}/test/server/mtls/openssl.cnf -days 365
20+
openssl req -x509 -keyout ${REPOSITORY_ROOT_DIR}/test/credentials/pg_client_key.pem -out ${REPOSITORY_ROOT_DIR}/test/credentials/pg_client_cert.pem -config ${STACKQL_CORE_DIR}/test/server/mtls/openssl.cnf -days 365
21+
openssl req -x509 -keyout ${REPOSITORY_ROOT_DIR}/test/credentials/pg_rubbish_key.pem -out ${REPOSITORY_ROOT_DIR}/test/credentials/pg_rubbish_cert.pem -config ${STACKQL_CORE_DIR}/test/server/mtls/openssl.cnf -days 365

‎cicd/scripts/local/11-run-mocked.sh

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env bash
2+
3+
>&2 echo "requires all of requirements.txt"
4+
5+
SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
6+
7+
source "${SCRIPT_DIR}/context.sh"
8+
9+
cd "${REPOSITORY_ROOT_DIR}"
10+
11+
source "${REPOSITORY_ROOT_DIR}/.venv/bin/activate"
12+
13+
export PYTHONPATH="${PYTHONPATH}:${REPOSITORY_ROOT_DIR}/test/python"
14+
15+
robot -d test/robot/reports/mocked test/robot/cli/mocked
16+

‎cicd/scripts/local/99-clean.sh

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#! /usr/bin/env bash
2+
3+
>&2 echo "cleaning dependent repositories from local file system"
4+
5+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
6+
7+
REPOSITORY_ROOT_DIR="$(realpath ${SCRIPT_DIR}/../../..)"
8+
9+
cd "${REPOSITORY_ROOT_DIR}"
10+
11+
rm -rf stackql-core || true
12+
rm -rf stackql-any-sdk || true
13+

‎cicd/scripts/local/context.sh

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/usr/bin/env bash
2+
3+
4+
SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
5+
6+
export REPOSITORY_ROOT_DIR="$(realpath ${SCRIPT_DIR}/../../..)"
7+
8+
export STACKQL_CORE_DIR="${STACKQL_CORE_DIR:-"${REPOSITORY_ROOT_DIR}/stackql-core}"}"
9+
10+
11+
checkPoetry () {
12+
if ! command -v poetry &> /dev/null
13+
then
14+
>&2 echo "Poetry is not installed. Please install it first."
15+
exit 1
16+
fi
17+
}

‎cicd/scripts/local/local_scripts.md

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
2+
3+
Here is a typical example, from the root oif this repository, assuming you have the core repository locally at `../stackql-devel`:
4+
5+
```bash
6+
7+
env STACKQL_CORE_DIR="$(realpath ../stackql-devel)" cicd/scripts/local/02-build-stackql-test-package.sh
8+
9+
env STACKQL_CORE_DIR="$(realpath ../stackql-devel)" cicd/scripts/local/03-install-stackql-test-package.sh
10+
11+
env STACKQL_CORE_DIR="$(realpath ../stackql-devel)" cicd/scripts/local/04-setup-testing-dependencies.sh
12+
13+
env GCP_SERVICE_ACCOUNT_KEY="$(cat test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json)" cicd/scripts/local/11-run-mocked.sh
14+
15+
```

‎cicd/testing-requirements.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
PyYaml==6.0.2
2+
requests==2.32.3
3+
robotframework==7.0.1
4+
tabulate==0.9.0

0 commit comments

Comments
 (0)