diff --git a/.github/workflows/ceph-s3-tests.yaml b/.github/workflows/ceph-s3-tests.yaml index 202e71a725..2c6cd4915e 100644 --- a/.github/workflows/ceph-s3-tests.yaml +++ b/.github/workflows/ceph-s3-tests.yaml @@ -74,6 +74,7 @@ jobs: kubectl logs job/noobaa-tests-s3 --tail 10000 -f if kubectl logs job/noobaa-tests-s3 | grep -q "Ceph Test Failed:"; then echo "At least one test failed!" + kubectl get pods exit 1 fi if [ ${TIMEOUT} ]; then diff --git a/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_guide.md b/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_guide.md index b4d957bf68..52fb644f72 100644 --- a/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_guide.md +++ b/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_guide.md @@ -89,6 +89,8 @@ kubectl apply -f src/test/system_tests/ceph_s3_tests/test_ceph_s3_job.yml kubectl logs job/noobaa-tests-s3 -f ``` +#### Skipped tests + We run all the tests except the tests that appear in the lists `src/test/system_tests/ceph_s3_tests/s3-tests-lists` if you would like to add or remove a test you can edit those files (and then repeat the steps starting from 'Build Core And Tester Images (Noobaa-Core)' above). Some tests are marked to be skipped in the tests code. usually because the enviorment doesn't support certain criteria. those tests will appear with a `[WARN]` tag and will be marked as "Test Skipped", for example: @@ -97,6 +99,14 @@ Some tests are marked to be skipped in the tests code. usually because the envio In the test code the function: `pytest.skip("")` will mark them to be skipped. +#### Boto3 tests +As mentioned in s3-tests repo (https://github.com/ceph/s3-tests#readme): +Most of the tests have both Boto3 and Boto2 versions. Tests written in Boto2 are in the s3tests directory. Tests written in Boto3 are located in the s3test_boto3 directory. + +In order to only run the boto3 version, tests in the boto directory run with `-- -m 'not fails_on_rgw`: + +`S3TEST_CONF=your.conf tox -- -m 'not fails_on_rgw' s3tests_boto3/functional::test_name` + ## Run a Single Ceph S3 Test ### 1) Prerequisites: @@ -144,16 +154,17 @@ Run the script that will create the necessary accounts in noobaa and update the node ./src/test/system_tests/ceph_s3_tests/test_ceph_s3_config_setup.js ``` -Note: If you want to ignore PythonDeprecationWarnings add the following flag to the test command (which will then ignore all Python warnings, so keep that in mind): +#### Disable pytest warnings +If you want to disable summary warnings add the following flag to the test command: `-- --disable-pytest-warnings` -note that every flag that comes after `--` is past to pytest from tox. so if there is already `--` in the command just put `--disable-pytest-warnings` as part of the flags after it, no need to add another `--` notation. +Note that every flag that comes after `--` is passed to pytest from tox. so if there is already `--` in the command just put `--disable-pytest-warnings` as part of the flags after it, no need to add another `--` notation. for example to add --disable-pytest-warnings to the command: -`S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini -- -m 'not fails_on_aws' ${PWD}/src/test/system_tests/ceph_s3_tests/s3-tests/s3tests_boto3/functional/test_s3.py::test_account_usage` +`S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini -- -m 'not fails_on_rgw' ${PWD}/src/test/system_tests/ceph_s3_tests/s3-tests/s3tests_boto3/functional/test_s3.py::test_account_usage` it should be: -`S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini -- -m 'not fails_on_aws --disable-pytest-warnings ${PWD}/src/test/system_tests/ceph_s3_tests/s3-tests/s3tests_boto3/functional/test_s3.py::test_account_usage` +`S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini -- -m 'not fails_on_rgw --disable-pytest-warnings ${PWD}/src/test/system_tests/ceph_s3_tests/s3-tests/s3tests_boto3/functional/test_s3.py::test_account_usage` ### 5) Run a Test (Inside The Tester Pod) To run a test, from noobaa working directory: @@ -163,7 +174,10 @@ S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf This should run the test on the noobaa deployment we've set up. #### Test Name -You can find a list of tests in the doc inside the file `ceph_s3_tests_list_single_test.txt`. Please notice that the test name has a certain structure are separated with `/`, the files end with the extention `.py` and the function to run (usually with a prefix `test_`) appears after the `::` sign. +You can find a list of tests in the doc inside the file `ceph_s3_tests_list_single_test.txt`. Please notice that the test name has a certain structure: directories are separated with `/`, the files end with the extention `.py` and the function to run (usually with a prefix `test_`) appears after the `::` sign. + +in case the test name is incorrect, for example if you add `:` instead of `::` to the test name, the command will fail. +the error will be `file or directory not found` and pytest will exit with error code 4 (which means "pytest command line usage error") ## Debug a Single Test (Inside The Tester Pod) ### 1) Prerequisites: @@ -186,13 +200,13 @@ Since the file `./src/test/system_tests/ceph_s3_tests/s3-tests/s3tests_boto3/fun 5) Build the tester image again, deploy noobaa, and run the test (repeat the steps starting from 'Build Core And Tester Images (Noobaa-Core)' above). #### B. Temporary change - this change will be saved in the file inside the container, useful when you need a small change. -You can edit the test by going to the test file and editing the test function. e.g. if you are working on test `s3tests_boto3.functional.test_s3:test_set_bucket_tagging` then you should `vi ./src/test/system_tests/ceph_s3_tests/s3-tests/s3tests_boto3/functional/test_s3.py` and search for the function `test_set_bucket_tagging`. -### Compare to AWS Response (Inside Tester Pod) +You can edit a test by going to the test file and editing the test function. e.g. if you are working on test `s3tests_boto3.functional.test_s3:test_set_bucket_tagging` then you should `vi ./src/test/system_tests/ceph_s3_tests/s3-tests/s3tests_boto3/functional/test_s3.py` and search for the function `test_set_bucket_tagging`. +## Compare to AWS Response (Inside Tester Pod) Prerequisites: Following the 'Run a Single Ceph S3 Test' steps until 'Deploy The Tester Deployment (Noobaa-Core Tab)'. In this section we will do some manual changes that will allow you to check AWS response for a specific test (tests that do not use neither ACL nor tenant group). -1) copy configuration file: +1) Copy configuration file - this will allow us to run a test on AWS and then back to NooBaa just by changing the configuration file (we would have 2 configuration files: `test_ceph_s3_config.conf` and `test_ceph_s3_config_aws.conf`): ```bash cp src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf src/test/system_tests/ceph_s3_tests/test_ceph_s3_config_aws.conf ``` @@ -228,13 +242,29 @@ CEPH TEST SUMMARY: Suite contains 812, ran 387 tests, Passed: 387, Skipped: 0, F Following the 'Run a Single Ceph S3 Test' steps. ### 1) Test Pass -For example: `s3tests_boto3/functional/test_s3.py::test_basic_key_count` -![alt text](images/tox_test_success.png) +For example: +```bash +S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini -- --disable-pytest-warnings ${PWD}/src/test/system_tests/ceph_s3_tests/s3-tests/s3tests/functional/test_headers.py::test_bucket_create_contentlength_none +``` +![test pass screenshot](images/tox_test_pass.png) -note that there is the warning: +Note that there is the warning: `WARNING: could not copy distfile to //.tox/distshare` this warning is for tox to use the same dependancies between projects. this feature is depricated and not used on this project. in order to remove the warning you can modify src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini to include the following line in the `[tox]` section: `distshare = /root/node_modules/noobaa-core/.tox/distshare` ### 2) Test Fail -For example: `s3tests_boto3/functional/test_s3::test_account_usage.py` -![alt text](images/tox_test_failed.png) +For example: +```bash +S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini -- --disable-pytest-warnings ${PWD}/src/test/system_tests/ceph_s3_tests/s3-tests/s3tests_boto3/functional/test_s3.py::test_account_usage +``` +![test failed screenshot](images/tox_test_failed.png) + +### 3) Test Skipped +```bash +S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini -- --disable-pytest-warnings ${PWD}/src/test/system_tests/ceph_s3_tests/s3-tests/s3tests_boto3/functional/test_s3.py::test_bucket_get_location +``` +![test skipped screenshot](images/tox_test_skipped.png) + +notice that even though test commands succeeded the test itself was skipped. the test prints `1 skipped` meaning one test was skipped + + diff --git a/docs/dev_guide/ceph_s3_tests/images/tox_test_pass.png b/docs/dev_guide/ceph_s3_tests/images/tox_test_pass.png new file mode 100644 index 0000000000..0881b95588 Binary files /dev/null and b/docs/dev_guide/ceph_s3_tests/images/tox_test_pass.png differ diff --git a/docs/dev_guide/ceph_s3_tests/images/tox_test_skipped.png b/docs/dev_guide/ceph_s3_tests/images/tox_test_skipped.png new file mode 100644 index 0000000000..d3810e582e Binary files /dev/null and b/docs/dev_guide/ceph_s3_tests/images/tox_test_skipped.png differ diff --git a/docs/dev_guide/ceph_s3_tests/images/tox_test_success.png b/docs/dev_guide/ceph_s3_tests/images/tox_test_success.png deleted file mode 100644 index 9b59443680..0000000000 Binary files a/docs/dev_guide/ceph_s3_tests/images/tox_test_success.png and /dev/null differ diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt index aa6a0ed676..a06d24d63c 100644 --- a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt +++ b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt @@ -76,3 +76,4 @@ s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_rgw_chunk_size s3tests_boto3/functional/test_s3.py::test_get_object_torrent s3tests_boto3/functional/test_s3select.py::test_count_json_operation s3tests_boto3/functional/test_s3select.py::test_column_sum_min_max +s3tests_boto3/functional/test_s3.py::test_multipart_upload_resend_part diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_s3.js b/src/test/system_tests/ceph_s3_tests/test_ceph_s3.js index f48c974d8a..592c25c704 100644 --- a/src/test/system_tests/ceph_s3_tests/test_ceph_s3.js +++ b/src/test/system_tests/ceph_s3_tests/test_ceph_s3.js @@ -15,7 +15,7 @@ const dbg = require('../../../util/debug_module')(__filename); dbg.set_process_name('test_ceph_s3'); const argv = require('minimist')(process.argv.slice(2)); delete argv._; -const { S3_CEPH_TEST_SIGV4, CEPH_TEST, DEFAULT_NUMBER_OF_WORKERS, TOX_ARGS } = require('./test_ceph_s3_constants.js'); +const { CEPH_TEST, DEFAULT_NUMBER_OF_WORKERS, TOX_ARGS, AWS4_TEST_ENDING } = require('./test_ceph_s3_constants.js'); const testing_status = { pass: [], @@ -65,7 +65,7 @@ async function run_s3_tests() { console.info(`CEPH TEST SUMMARY: Suite contains ${testing_status.total}, ran ${testing_status.pass.length + testing_status.fail.length + testing_status.skip.length} tests, Passed: ${testing_status.pass.length}, Skipped: ${testing_status.skip.length}, Failed: ${testing_status.fail.length}`); if (testing_status.skip.length) { - console.warn(`CEPH TEST SUMMARY: ${testing_status.skip.length} skipped tests \n${testing_status.skip.join('\n')}`); + console.warn(`CEPH TEST SKIPPED TESTS SUMMARY: ${testing_status.skip.length} skipped tests \n${testing_status.skip.join('\n')}`); } if (testing_status.fail.length) { console.error(`CEPH TEST FAILED TESTS SUMMARY: ${testing_status.fail.length} failed tests \n${testing_status.fail.join('\n')}`); @@ -99,34 +99,35 @@ async function test_worker() { } } -async function run_single_test(test_name) { +async function run_single_test(test) { let ceph_args = `S3TEST_CONF=${process.cwd()}/${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`; - if (S3_CEPH_TEST_SIGV4.includes(test_name)) { + if (test.endsWith(AWS4_TEST_ENDING)) { ceph_args += ` S3_USE_SIGV4=true`; } let base_cmd = `${ceph_args} tox ${TOX_ARGS}`; - if (!S3_CEPH_TEST_OUT_OF_SCOPE_REGEXP.test(test_name)) { + if (!S3_CEPH_TEST_OUT_OF_SCOPE_REGEXP.test(test)) { try { - if (test_name.includes('boto')) { - base_cmd = `${ceph_args} tox ${TOX_ARGS} -- -m 'not fails_on_aws'`; + if (test.includes('boto')) { + base_cmd = `${ceph_args} tox ${TOX_ARGS} -- -m 'not fails_on_rgw'`; } - const res = await os_utils.exec(`${base_cmd} ${process.cwd()}/${CEPH_TEST.test_dir}${CEPH_TEST.s3_test_dir}${test_name}`, { ignore_rc: false, return_stdout: true }); - if (res.indexOf('skipped') >= 0) { - console.warn('Test skipped:', test_name); - testing_status.skip.push(test_name); + const full_test_command = `${base_cmd} ${process.cwd()}/${CEPH_TEST.test_dir}${CEPH_TEST.s3_test_dir}${test}`; + const res = await os_utils.exec(full_test_command, { ignore_rc: false, return_stdout: true }); + if (res.includes('skipped')) { + console.warn('Test skipped:', test); + testing_status.skip.push(test); } else { - console.info('Test Passed:', test_name); - testing_status.pass.push(test_name); + console.info('Test Passed:', test); + testing_status.pass.push(test); } } catch (err) { // tox will exit with code 1 on error regardless of pytest exit code. pytest exit code 5 means no tests ran. - // can happen when 'not fails_on_aws' flag is on for some tests (there are no boto3 tests for the test) - if (err.stdout.indexOf("exited with code 5") >= 0) { - console.warn('Test skipped:', test_name); - testing_status.skip.push(test_name); + // can happen when 'not fails_on_rgw' flag is on for some tests (there are no boto3 tests for the test) + if (err.stdout.includes("exited with code 5")) { + console.warn('Test skipped:', test); + testing_status.skip.push(test); } else { - console.error('Test Failed:', test_name); - testing_status.fail.push(test_name); + console.error('Test Failed:', test); + testing_status.fail.push(test); } } } diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_s3_constants.js b/src/test/system_tests/ceph_s3_tests/test_ceph_s3_constants.js index e9e37205a7..723ae0d1f5 100644 --- a/src/test/system_tests/ceph_s3_tests/test_ceph_s3_constants.js +++ b/src/test/system_tests/ceph_s3_tests/test_ceph_s3_constants.js @@ -1,64 +1,6 @@ /* Copyright (C) 2022 NooBaa */ "use strict"; -const S3_CEPH_TEST_SIGV4 = [ - 'check_can_test_multiregion', - 'test_bucket_create_bad_amz_date_after_today_aws4', - 'test_bucket_create_bad_amz_date_before_epoch_aws4', - 'test_bucket_create_bad_amz_date_before_today_aws4', - 'test_bucket_create_bad_amz_date_empty_aws4', - 'test_bucket_create_bad_amz_date_invalid_aws4', - 'test_bucket_create_bad_amz_date_none_aws4', - 'test_bucket_create_bad_amz_date_unreadable_aws4', - 'test_bucket_create_bad_authorization_invalid_aws4', - 'test_bucket_create_bad_date_after_today_aws4', - 'test_bucket_create_bad_date_before_epoch_aws4', - 'test_bucket_create_bad_date_before_today_aws4', - 'test_bucket_create_bad_date_empty_aws4', - 'test_bucket_create_bad_date_invalid_aws4', - 'test_bucket_create_bad_date_none_aws4', - 'test_bucket_create_bad_date_unreadable_aws4', - 'test_bucket_create_bad_ua_empty_aws4', - 'test_bucket_create_bad_ua_none_aws4', - 'test_bucket_create_bad_ua_unreadable_aws4', - 'test_object_create_bad_amz_date_after_end_aws4', - 'test_object_create_bad_amz_date_after_today_aws4', - 'test_object_create_bad_amz_date_before_epoch_aws4', - 'test_object_create_bad_amz_date_before_today_aws4', - 'test_object_create_bad_amz_date_empty_aws4', - 'test_object_create_bad_amz_date_invalid_aws4', - 'test_object_create_bad_amz_date_none_aws4', - 'test_object_create_bad_amz_date_unreadable_aws4', - 'test_object_create_bad_authorization_incorrect_aws4', - 'test_object_create_bad_authorization_invalid_aws4', - 'test_object_create_bad_contentlength_mismatch_below_aws4', - 'test_object_create_bad_date_after_end_aws4', - 'test_object_create_bad_date_after_today_aws4', - 'test_object_create_bad_date_before_epoch_aws4', - 'test_object_create_bad_date_before_today_aws4', - 'test_object_create_bad_date_empty_aws4', - 'test_object_create_bad_date_invalid_aws4', - 'test_object_create_bad_date_none_aws4', - 'test_object_create_bad_date_unreadable_aws4', - 'test_object_create_bad_md5_invalid_garbage_aws4', - 'test_object_create_bad_ua_empty_aws4', - 'test_object_create_bad_ua_none_aws4', - 'test_object_create_bad_ua_unreadable_aws4', - 'test_object_create_missing_signed_custom_header_aws4', - 'test_object_create_missing_signed_header_aws4', - 'test_object_raw_get_x_amz_expires_not_expired', - 'test_object_raw_get_x_amz_expires_out_max_range', - 'test_object_raw_get_x_amz_expires_out_positive_range', - 'test_object_raw_get_x_amz_expires_out_range_zero', - 'test_region_bucket_create_master_access_remove_secondary', - 'test_region_bucket_create_secondary_access_remove_master', - 'test_region_copy_object', - 'test_sse_kms_barb_transfer_13b', - 'test_sse_kms_barb_transfer_1b', - 'test_sse_kms_barb_transfer_1kb', - 'test_sse_kms_barb_transfer_1MB' -]; - const CEPH_TEST = { test_dir: 'src/test/system_tests/ceph_s3_tests/', s3_test_dir: 's3-tests/', @@ -82,8 +24,9 @@ const DEFAULT_NUMBER_OF_WORKERS = 5; //5 was the number of workers in the previo const TOX_ARGS = `-c ${CEPH_TEST.test_dir}${CEPH_TEST.s3_test_dir}${CEPH_TEST.tox_config}`; +const AWS4_TEST_ENDING = '_aws4'; -exports.S3_CEPH_TEST_SIGV4 = S3_CEPH_TEST_SIGV4; exports.CEPH_TEST = CEPH_TEST; exports.DEFAULT_NUMBER_OF_WORKERS = DEFAULT_NUMBER_OF_WORKERS; exports.TOX_ARGS = TOX_ARGS; +exports.AWS4_TEST_ENDING = AWS4_TEST_ENDING;