From 1b0bc85d5fe23f93c6ba618b896b34ef3e04e816 Mon Sep 17 00:00:00 2001 From: nadav mizrahi Date: Mon, 17 Apr 2023 16:58:05 +0300 Subject: [PATCH 1/4] add condition to check limit for number of deleted objects Signed-off-by: nadav mizrahi --- src/endpoint/s3/ops/s3_post_bucket_delete.js | 7 +++++++ .../ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt | 1 - 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/endpoint/s3/ops/s3_post_bucket_delete.js b/src/endpoint/s3/ops/s3_post_bucket_delete.js index 6791eadb3e..cd4c641402 100644 --- a/src/endpoint/s3/ops/s3_post_bucket_delete.js +++ b/src/endpoint/s3/ops/s3_post_bucket_delete.js @@ -3,6 +3,7 @@ const _ = require('lodash'); const dbg = require('../../../util/debug_module')(__filename); +const S3Error = require('../s3_errors').S3Error; /** * http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html @@ -15,6 +16,12 @@ async function post_bucket_delete(req) { key: obj.Key && obj.Key[0], version_id: obj.VersionId && obj.VersionId[0], })); + + if (objects.length > 1000) { + dbg.error('The request can not contain a list of more than 1000 keys'); + throw new S3Error(S3Error.MalformedXML); + } + dbg.log3('post_bucket_delete: objects', objects); const reply = await req.object_sdk.delete_multiple_objects({ diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt index a0c70d22aa..d7fd3580d7 100644 --- a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt +++ b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt @@ -9,7 +9,6 @@ s3tests_boto3.functional.test_s3.test_bucket_list_prefix_basic s3tests_boto3.functional.test_s3.test_bucket_list_prefix_delimiter_basic s3tests_boto3.functional.test_s3.test_account_usage s3tests_boto3.functional.test_s3.test_head_bucket_usage -s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit s3tests_boto3.functional.test_s3.test_post_object_invalid_signature s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition From 7ed8400aca701d272d8b05f3b78717fae1d11ac2 Mon Sep 17 00:00:00 2001 From: nadav mizrahi Date: Mon, 17 Apr 2023 17:09:14 +0300 Subject: [PATCH 2/4] change error for invalid algorithm to invalidArgument Signed-off-by: nadav mizrahi --- src/endpoint/s3/s3_utils.js | 4 ++-- .../ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/endpoint/s3/s3_utils.js b/src/endpoint/s3/s3_utils.js index dc34e988d8..8a71846dde 100644 --- a/src/endpoint/s3/s3_utils.js +++ b/src/endpoint/s3/s3_utils.js @@ -227,9 +227,9 @@ function parse_sse(req) { } } - if (algorithm !== 'AES256' && algorithm !== 'aws:kms') throw new S3Error(S3Error.InvalidDigest); + if (algorithm !== 'AES256' && algorithm !== 'aws:kms') throw new S3Error(S3Error.InvalidArgument); - if (algorithm === 'aws:kms' && !kms_key_id) throw new S3Error(S3Error.InvalidDigest); + if (algorithm === 'aws:kms' && !kms_key_id) throw new S3Error(S3Error.InvalidArgument); // const md5_regex = new RegExp('/^[a-f0-9]{32}$/'); // if (kms_key_id && !md5_regex.test(kms_key_id)) throw new S3Error(S3Error.InvalidDigest); diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt index d7fd3580d7..5898ed58aa 100644 --- a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt +++ b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt @@ -33,7 +33,6 @@ s3tests_boto3.functional.test_s3.test_lifecycle_expiration_header_tags_head s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_c_s3 s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_c_kms s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_s3_kms -s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_bad_enc_kms s3tests_boto3.functional.test_s3.test_bucket_policy_put_obj_s3_noenc s3tests_boto3.functional.test_s3.test_bucket_policy_put_obj_s3_kms s3tests_boto3.functional.test_s3.test_bucket_policy_put_obj_kms_noenc From c149641e368f2d3d67132bffb73a5b86e057fed2 Mon Sep 17 00:00:00 2001 From: nadav mizrahi Date: Sun, 23 Apr 2023 14:58:08 +0300 Subject: [PATCH 3/4] fix testing jobs out of scope tests regex to match against full name Signed-off-by: nadav mizrahi --- .../ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt | 4 +++- .../ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt | 7 +++++++ src/test/system_tests/ceph_s3_tests/test_ceph_s3.js | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt index 8ab22f58f4..6a4ee91db0 100644 --- a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt +++ b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt @@ -71,6 +71,7 @@ s3tests_boto3.functional.test_headers.test_bucket_create_bad_date_invalid_aws2 s3tests_boto3.functional.test_headers.test_bucket_create_bad_date_none_aws2 s3tests_boto3.functional.test_headers.test_bucket_create_bad_date_empty_aws2 s3tests_boto3.functional.test_headers.test_bucket_create_bad_date_before_epoch_aws2 +s3tests.functional.test_headers.test_object_acl_create_contentlength_none s3tests_boto3.functional.test_s3.test_bucket_listv2_delimiter_prefix s3tests_boto3.functional.test_s3.test_bucket_listv2_delimiter_prefix_underscore s3tests_boto3.functional.test_s3.test_bucket_list_delimiter_unreadable @@ -163,6 +164,7 @@ s3tests_boto3.functional.test_s3.test_object_acl_canned_bucketownerread s3tests_boto3.functional.test_s3.test_object_acl_canned_authenticatedread s3tests_boto3.functional.test_s3.test_object_acl_canned_bucketownerfullcontrol s3tests_boto3.functional.test_s3.test_object_acl_full_control_verify_owner +s3tests_boto3.functional.test_s3.test_object_acl_full_control_verify_attributes s3tests_boto3.functional.test_s3.test_object_acl s3tests_boto3.functional.test_s3.test_object_acl_write s3tests_boto3.functional.test_s3.test_object_acl_writeacp @@ -178,7 +180,7 @@ s3tests_boto3.functional.test_s3.test_bucket_acl_no_grants s3tests_boto3.functional.test_s3.test_bucket_header_acl_grants s3tests_boto3.functional.test_s3.test_object_header_acl_grants s3tests_boto3.functional.test_s3.test_bucket_acl_grant_email -s3tests_boto3.functional.test_s3.test_bucket_acl_grant_email_notexist +s3tests_boto3.functional.test_s3.test_bucket_acl_grant_email_not_exist s3tests_boto3.functional.test_s3.test_bucket_acl_revoke_all s3tests_boto3.functional.test_s3.test_logging_toggle s3tests_boto3.functional.test_s3.test_access_bucket_private_object_private diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt index 5898ed58aa..82cb34b1ca 100644 --- a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt +++ b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt @@ -15,6 +15,13 @@ s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field s3tests_boto3.functional.test_s3.test_post_object_expired_policy s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value +s3tests_boto3.functional.test_s3.test_lifecycle_get_no_id +s3tests_boto3.functional.test_s3.test_object_lock_put_obj_lock_invalid_mode +s3tests_boto3.functional.test_s3.test_object_lock_get_obj_retention_iso8601 +s3tests_boto3.functional.test_s3.test_object_lock_delete_object_with_retention_and_marker +s3tests_boto3.functional.test_s3.test_multipart_upload_on_a_bucket_with_policy +s3tests_boto3.functional.test_s3.test_multipart_upload_small +s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key s3tests_boto3.functional.test_s3.test_bucket_create_exists s3tests_boto3.functional.test_s3.test_bucket_create_exists_nonowner s3tests_boto3.functional.test_s3.test_bucket_recreate_overwrite_acl diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_s3.js b/src/test/system_tests/ceph_s3_tests/test_ceph_s3.js index 6ad2d86a3b..fa4189e38a 100644 --- a/src/test/system_tests/ceph_s3_tests/test_ceph_s3.js +++ b/src/test/system_tests/ceph_s3_tests/test_ceph_s3.js @@ -28,7 +28,7 @@ let tests_list; const OUT_OF_SCOPE_TESTS = create_out_of_scope_tests_list() || []; //Regexp match will be tested per each entry -const S3_CEPH_TEST_OUT_OF_SCOPE_REGEXP = new RegExp(`(${OUT_OF_SCOPE_TESTS.join(')|(')})`); +const S3_CEPH_TEST_OUT_OF_SCOPE_REGEXP = new RegExp(`(${OUT_OF_SCOPE_TESTS.join('\\b)|(')}\\b)`); const S3_CEPH_TEST_STEMS_REGEXP = new RegExp(`(${S3_CEPH_TEST_STEMS.join(')|(')})`); async function main() { From aecb4f7164ac9c8a763646b0bc9ee78f0c70a97d Mon Sep 17 00:00:00 2001 From: nadav mizrahi Date: Thu, 27 Apr 2023 16:07:14 +0300 Subject: [PATCH 4/4] add test_lifecycle_set_date and test_lifecycle_transition_set_invalid_date to tests pending list Signed-off-by: nadav mizrahi --- .../ceph_s3_tests/ceph_s3_tests_pending_list_status.md | 4 +++- .../ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_pending_list_status.md b/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_pending_list_status.md index bd342ef66c..24466ffb15 100644 --- a/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_pending_list_status.md +++ b/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_pending_list_status.md @@ -34,4 +34,6 @@ Attached a table with tests that where investigated and their status (this table | test_generate_projection | Faulty Test | [509](https://github.com/ceph/s3-tests/issues/509) | | | test_alias_cyclic_refernce | Faulty Test | | Stops execution after failure is returned instead of parsing error. I'm not opening issue as it might be related to outdated tests. | | test_schema_definition | Faulty Test | | Same as test_alias_cyclic_refernce | -| test_progress_expressions | Faulty Test | [508](https://github.com/ceph/s3-tests/issues/508) | | \ No newline at end of file +| test_progress_expressions | Faulty Test | [508](https://github.com/ceph/s3-tests/issues/508) | | +| test_lifecycle_set_date | Faulty Test | [510](https://github.com/ceph/s3-tests/issues/510) | | +| test_lifecycle_transition_set_invalid_date | Not Implemented | | added because of the following PR [7270](https://github.com/noobaa/noobaa-core/pull/7270#discussion_r1175123422) | \ No newline at end of file diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt index 82cb34b1ca..c4dfcfa015 100644 --- a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt +++ b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt @@ -37,6 +37,7 @@ s3tests_boto3.functional.test_s3.test_lifecycle_set_date s3tests_boto3.functional.test_s3.test_lifecycle_expiration_header_put s3tests_boto3.functional.test_s3.test_lifecycle_expiration_header_head s3tests_boto3.functional.test_s3.test_lifecycle_expiration_header_tags_head +s3tests_boto3.functional.test_s3.test_lifecycle_transition_set_invalid_date s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_c_s3 s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_c_kms s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_s3_kms