Skip to content

Commit 287f9f5

Browse files
leo-cai-timeplusleo-cai-timeplus
committed
add test cases
1 parent b414464 commit 287f9f5

File tree

11 files changed

+381
-38
lines changed

11 files changed

+381
-38
lines changed

‎.github/workflows/compatibility_test.yml

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: compatibility_test
2-
2+
run-name: Compatibility Test [${{ (github.event.inputs.source == '' || github.event.inputs.source == github.event.inputs.target) && format('Upload {0}', github.event.inputs.target) || format('{0} -> {1}', github.event.inputs.source, github.event.inputs.target) }}]
33
on:
44
workflow_dispatch:
55
inputs:
@@ -31,21 +31,21 @@ jobs:
3131
3232
pip install --upgrade pip
3333
34-
# FIXME: remove this line after pyyaml community fixed install bug
35-
pip install pyyaml==5.3.1
36-
3734
# FIXME(yokofly): docker 7.0.0 introduce a breaking change
3835
# https://github.com/docker/docker-py/issues/3194
3936
pip install docker==6.1.3
4037
41-
pip install -r helpers/requirements.txt
38+
pip install -r test_compatibility/requirements.txt
4239
43-
bash test_compatibility/prepare_data.sh
44-
bash test_compatibility/basic_tests.sh
40+
timeout --foreground 10m bash test_compatibility/prepare_data.sh
41+
timeout --foreground 10m bash test_compatibility/basic_tests.sh
4542
4643
cd $GITHUB_WORKSPACE
4744
tar -zcvf $PROTON_VERSION.tar.gz data
4845
aws s3 cp --no-progress $PROTON_VERSION.tar.gz s3://tp-internal/proton/compatibility/oss/$ARCH/
46+
47+
cd $GITHUB_WORKSPACE/tests/stream
48+
timeout --foreground 10m bash test_compatibility/extra_tests.sh
4949
steps:
5050
- name: display command
5151
run: |
@@ -75,18 +75,15 @@ jobs:
7575
source env/bin/activate
7676
7777
pip install --upgrade pip
78-
79-
# FIXME: remove this line after pyyaml community fixed install bug
80-
pip install pyyaml==5.3.1
81-
78+
8279
# FIXME(yokofly): docker 7.0.0 introduce a breaking change
8380
# https://github.com/docker/docker-py/issues/3194
8481
pip install docker==6.1.3
82+
83+
pip install -r test_compatibility/requirements.txt
8584
86-
pip install -r helpers/requirements.txt
87-
88-
bash test_compatibility/basic_tests.sh
89-
bash test_compatibility/extra_tests.sh
85+
timeout --foreground 10m bash test_compatibility/basic_tests.sh
86+
timeout --foreground 10m bash test_compatibility/extra_tests.sh
9087
steps:
9188
- name: display command
9289
run: |
@@ -182,4 +179,4 @@ jobs:
182179
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
183180
TIMEPLUS_ADDRESS: ${{ secrets.TIMEPLUS_ADDRESS }}
184181
TIMEPLUS_API_KEY: ${{ secrets.TIMEPLUS_API_KEY }}
185-
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}
182+
TIMEPLUS_WORKSPACE: ${{ secrets.TIMEPLUS_WORKSPACE }}

‎tests/stream/test_compatibility/basic_tests.sh

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
1-
docker-compose -f test_compatibility/docker-compose.yaml up -d
1+
set -e
2+
CUR_DIR="$GITHUB_WORKSPACE/tests/stream/test_compatibility"
3+
docker-compose -f "$CUR_DIR/configs/docker-compose.yaml" up -d
4+
docker ps
25
sleep 5
3-
docker exec proton-server proton client -nm -q "select x from example where _tp_time > earliest_ts() limit 3;"
4-
docker exec proton-server proton client -nm -q "select x from example_external limit 3 settings seek_to='earliest';"
5-
docker-compose -f test_compatibility/docker-compose.yaml down
6+
export TEST_DIR="$CUR_DIR/basic_tests"
7+
python $CUR_DIR/run_compatibility_tests.py
8+
docker exec proton-server pkill proton-server
9+
sleep 10
10+
docker-compose -f "$CUR_DIR/configs/docker-compose.yaml" down -v

‎tests/stream/test_compatibility/basic_tests/example.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
example:
2+
description: This is an example of test_suit
3+
steps:
4+
- cmd: select 1;
5+
query_time: 3
6+
wait: 1
7+
query_id: '0000-0000'
8+
expect:
9+
- [1]

‎tests/stream/test_compatibility/basic_tests/materialized_view.yaml

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
alter_stream_add_column:
2+
description: query from a materialized view on stream, and then alter the stream by adding column
3+
steps:
4+
- cmd: |
5+
select (* except _tp_time) from table(test_1_mv) order by col1;
6+
query_time: 3
7+
expect:
8+
- [ 1, 1.1, 'a1' ]
9+
- [ 2, 2.1, 'b2' ]
10+
- [ 3, 3.1, 'c3' ]
11+
- [ 4, 4.1, 'd4' ]
12+
- [ 5, 5.1, 'e5' ]
13+
- [ 6, 6.1, 'f6' ]
14+
- cmd: |
15+
select (* except _tp_time) from test_1_mv where _tp_time > earliest_ts() order by col1;
16+
query_time: 3
17+
expect:
18+
- [ 1, 1.1, 'a1' ]
19+
- [ 2, 2.1, 'b2' ]
20+
- [ 3, 3.1, 'c3' ]
21+
- [ 4, 4.1, 'd4' ]
22+
- [ 5, 5.1, 'e5' ]
23+
- [ 6, 6.1, 'f6' ]
24+
25+
global_aggregation:
26+
description: a materialized view on stream global aggregation
27+
steps:
28+
- cmd: |
29+
select (* except _tp_time) from table(test_2_mv);
30+
query_time: 3
31+
expect:
32+
- [ 1.1, 6.1, 8 ]
33+
- cmd: |
34+
select (* except _tp_time) from test_2_mv where _tp_time > earliest_ts() limit 1;
35+
expect:
36+
- [ 1.1, 6.1, 8 ]
37+
38+
func_now:
39+
description: test for function now()
40+
steps:
41+
- cmd: |
42+
select (* except _tp_time) from table(test_3_mv) order by i;
43+
query_time: 3
44+
expect:
45+
- [ "any_value", 1 ]
46+
- [ "any_value", 2 ]
47+
- [ "any_value", 3 ]
48+
- [ "any_value", 4 ]

‎tests/stream/test_compatibility/docker-compose.yaml renamed to ‎tests/stream/test_compatibility/configs/docker-compose.yaml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,11 @@ services:
88
- $GITHUB_WORKSPACE/data/proton-redp/datas:/var/lib/proton
99
- $GITHUB_WORKSPACE/data/proton-redp/log:/var/log/proton-server
1010
ports:
11-
- "13218:3218" # HTTP Streaming
12-
- "18123:8123" # HTTP Snapshot
13-
- "18463:8463" # TCP Streaming
14-
- "15432:5432" # Postgres Snapshot
15-
- "17587:7587" # TCP Snapshot
11+
- "3218:3218" # HTTP Streaming
12+
- "8123:8123" # HTTP Snapshot
13+
- "8463:8463" # TCP Streaming
14+
- "5432:5432" # Postgres Snapshot
15+
- "7587:7587" # TCP Snapshot
1616
deploy:
1717
replicas: 1
1818
restart_policy:

‎tests/stream/test_compatibility/extra_tests.sh

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1-
docker-compose -f test_compatibility/docker-compose.yaml up -d
1+
set -e
2+
CUR_DIR="$GITHUB_WORKSPACE/tests/stream/test_compatibility"
3+
docker-compose -f "$CUR_DIR/configs/docker-compose.yaml" up -d
4+
docker ps
25
sleep 5
3-
docker exec proton-server proton client -nm -q "insert into example (x) values (4)(5)(6);"
4-
docker exec proton-server proton client -nm -q "select x from example where _tp_time > earliest_ts() limit 6;"
5-
docker exec proton-server proton client -nm -q "insert into example_external (x) values (8)(10)(12);"
6-
docker exec proton-server proton client -nm -q "select x from example_external limit 6 settings seek_to='earliest';"
7-
docker-compose -f test_compatibility/docker-compose.yaml down
6+
export TEST_DIR="$CUR_DIR/extra_tests"
7+
python $CUR_DIR/run_compatibility_tests.py
8+
docker exec proton-server pkill proton-server
9+
sleep 10
10+
docker-compose -f "$CUR_DIR/configs/docker-compose.yaml" down -v

‎tests/stream/test_compatibility/extra_tests/materialized_view.yaml

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
alter_stream_add_column:
2+
description: query from a materialized view on stream, and then alter the stream by adding column
3+
steps:
4+
- cmd: |
5+
insert into test_1_s (* except _tp_time)
6+
values
7+
(7, 7.1, 'g7', 47), (8, 8.1, 'h8', 48), (9, 9.1, 'i9', 49);
8+
wait: 2
9+
- cmd: |
10+
select (* except _tp_time) from table(test_1_mv) order by col1;
11+
query_time: 3
12+
expect:
13+
- [ 1, 1.1, 'a1' ]
14+
- [ 2, 2.1, 'b2' ]
15+
- [ 3, 3.1, 'c3' ]
16+
- [ 4, 4.1, 'd4' ]
17+
- [ 5, 5.1, 'e5' ]
18+
- [ 6, 6.1, 'f6' ]
19+
- [ 7, 7.1, 'g7' ]
20+
- [ 8, 8.1, 'h8' ]
21+
- [ 9, 9.1, 'i9' ]
22+
- cmd: |
23+
select (* except _tp_time) from test_1_mv where _tp_time > earliest_ts() order by col1 limit 9;
24+
query_time: 3
25+
expect:
26+
- [ 1, 1.1, 'a1' ]
27+
- [ 2, 2.1, 'b2' ]
28+
- [ 3, 3.1, 'c3' ]
29+
- [ 4, 4.1, 'd4' ]
30+
- [ 5, 5.1, 'e5' ]
31+
- [ 6, 6.1, 'f6' ]
32+
- [ 7, 7.1, 'g7' ]
33+
- [ 8, 8.1, 'h8' ]
34+
- [ 9, 9.1, 'i9' ]
35+
36+
global_aggregation:
37+
description: a materialized view on stream global aggregation
38+
steps:
39+
- cmd: |
40+
select (* except _tp_time) from table(test_2_mv);
41+
query_time: 3
42+
expect:
43+
- [ 1.1, 6.1, 8 ]
44+
- cmd: |
45+
select (* except _tp_time) from test_2_mv where _tp_time > earliest_ts();
46+
query_time: 3
47+
expect:
48+
- [ 1.1, 6.1, 8 ]
49+
- cmd: |
50+
insert into test_2_s (* except _tp_time)
51+
values
52+
(7, 7.1), (8, 8.1), (9, 9.1);
53+
wait: 3
54+
- cmd: |
55+
select (* except _tp_time) from table(test_2_mv) order by cnt_col2;
56+
query_time: 3
57+
expect:
58+
- [ 1.1, 6.1, 8 ]
59+
- [ 1.1, 9.1, 11 ]
60+
- cmd: |
61+
select (* except _tp_time) from test_2_mv where _tp_time > earliest_ts() order by cnt_col2;
62+
query_time: 3
63+
expect:
64+
- [ 1.1, 6.1, 8 ]
65+
- [ 1.1, 9.1, 11 ]
66+
67+
func_now:
68+
description: test for function now()
69+
steps:
70+
- cmd: |
71+
insert into test_3_s (* except _tp_time) values (5);
72+
wait: 3
73+
- cmd: |
74+
select (* except _tp_time) from table(test_3_mv) order by i;
75+
query_time: 3
76+
expect:
77+
- [ "any_value", 1 ]
78+
- [ "any_value", 2 ]
79+
- [ "any_value", 3 ]
80+
- [ "any_value", 4 ]
81+
- [ "any_value", 5 ]

‎tests/stream/test_compatibility/prepare_data.sh

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1-
docker-compose -f test_compatibility/docker-compose.yaml up -d
1+
set -e
2+
CUR_DIR="$GITHUB_WORKSPACE/tests/stream/test_compatibility"
3+
docker-compose -f "$CUR_DIR/configs/docker-compose.yaml" up -d
4+
docker ps
25
sleep 5
3-
docker exec proton-server proton client -nm -q "create stream example(x int);"
4-
docker exec proton-server proton client -nm -q "insert into example (x) values (1)(2)(3);"
5-
docker exec redpanda rpk topic create example_topic
6-
docker exec proton-server proton client -nm -q "create external stream example_external (x int) settings type='kafka', brokers='stream-store:9092',topic='example_topic',data_format = 'JSONEachRow';"
7-
docker exec proton-server proton client -nm -q "insert into example_external (x) values (2)(4)(6);"
8-
docker-compose -f test_compatibility/docker-compose.yaml down
6+
export TEST_DIR="$CUR_DIR/prepare_data"
7+
python $CUR_DIR/run_compatibility_tests.py
8+
docker exec proton-server pkill proton-server
9+
sleep 10
10+
docker-compose -f "$CUR_DIR/configs/docker-compose.yaml" down

‎tests/stream/test_compatibility/prepare_data/materialized_view.yaml

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
alter_stream_add_column:
2+
description: a materialized view on stream, and then alter the stream by adding column
3+
steps:
4+
- cmd: |
5+
create stream test_1_s (
6+
col1 int,
7+
col2 float,
8+
col3 string
9+
);
10+
- cmd: |
11+
create materialized view test_1_mv as (
12+
select
13+
col1, col2, col3
14+
from
15+
test_1_s
16+
);
17+
- cmd: |
18+
insert into test_1_s (* except _tp_time)
19+
values
20+
(1, 1.1, 'a1'), (2, 2.1, 'b2'), (3, 3.1, 'c3');
21+
- cmd: |
22+
alter stream test_1_s add column col4 uint64;
23+
- cmd: |
24+
insert into test_1_s (* except _tp_time)
25+
values
26+
(4, 4.1, 'd4', 44), (5, 5.1, 'e5', 45), (6, 6.1, 'f6', 46);
27+
28+
global_aggregation:
29+
description: a materialized view on stream global aggregation
30+
steps:
31+
- cmd: |
32+
create stream test_2_s (
33+
col1 int,
34+
col2 float64
35+
);
36+
- cmd: |
37+
create materialized view test_2_mv as(
38+
select
39+
min(col2) as min_col2,
40+
max(col2) as max_col2,
41+
count(col2) as cnt_col2
42+
from test_2_s
43+
);
44+
- cmd: |
45+
insert into test_2_s (* except _tp_time)
46+
values
47+
(1, 1.1), (2, 2.1), (3, 3.1), (4, 4.1), (5, 5.1), (6, 6.1), (3, 3.1), (4, 4.1);
48+
49+
func_now:
50+
description: test for function now()
51+
steps:
52+
- cmd: |
53+
create stream test_3_s (
54+
i int
55+
);
56+
- cmd: |
57+
create materialized view test_3_mv as (
58+
select
59+
now(), i
60+
from
61+
test_3_s
62+
);
63+
- cmd: |
64+
insert into test_3_s (* except _tp_time) values (1);
65+
- cmd: |
66+
insert into test_3_s (* except _tp_time) values (2);
67+
- cmd: |
68+
insert into test_3_s (* except _tp_time) values (3);
69+
- cmd: |
70+
insert into test_3_s (* except _tp_time) values (4);

‎tests/stream/test_compatibility/requirements.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
docker-compose==1.29.2
2+
proton-driver>=0.2.9
3+
ddt>=1.7.0
4+
pyyaml==5.3.1

0 commit comments

Comments
 (0)