Merge pull request #7270 from nadavMiz/ceph-test-changes

Multiple Fixes Following Failed S3 Tests
noobaa · May 31, 2023 · ecf338a · ecf338a
2 parents 0e664d1 + aecb4f7
commit ecf338a
Show file tree

Hide file tree

Showing 6 changed files with 24 additions and 7 deletions.
diff --git a/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_pending_list_status.md b/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_pending_list_status.md
@@ -34,4 +34,6 @@ Attached a table with tests that where investigated and their status (this table
 | test_generate_projection                                  | Faulty Test     | [509](https://github.com/ceph/s3-tests/issues/509)                    |                      |
 | test_alias_cyclic_refernce                                | Faulty Test     |                      | Stops execution after failure is returned instead of parsing error. I'm not opening issue as it might be related to outdated tests.                     |
 | test_schema_definition                                    | Faulty Test     |                      | Same as test_alias_cyclic_refernce |
-| test_progress_expressions                                 | Faulty Test     | [508](https://github.com/ceph/s3-tests/issues/508)                    |                      |
+| test_progress_expressions                                 | Faulty Test     | [508](https://github.com/ceph/s3-tests/issues/508)                    |                      |
+| test_lifecycle_set_date                                   | Faulty Test     | [510](https://github.com/ceph/s3-tests/issues/510)                    |                      |
+| test_lifecycle_transition_set_invalid_date                | Not Implemented |                      |    added because of the following PR [7270](https://github.com/noobaa/noobaa-core/pull/7270#discussion_r1175123422)   |
diff --git a/src/endpoint/s3/ops/s3_post_bucket_delete.js b/src/endpoint/s3/ops/s3_post_bucket_delete.js
@@ -3,6 +3,7 @@
 
 const _ = require('lodash');
 const dbg = require('../../../util/debug_module')(__filename);
+const S3Error = require('../s3_errors').S3Error;
 
 /**
  * http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
@@ -15,6 +16,12 @@ async function post_bucket_delete(req) {
         key: obj.Key && obj.Key[0],
         version_id: obj.VersionId && obj.VersionId[0],
     }));
+
+    if (objects.length > 1000) {
+        dbg.error('The request can not contain a list of more than 1000 keys');
+        throw new S3Error(S3Error.MalformedXML);
+    }
+
     dbg.log3('post_bucket_delete: objects', objects);
 
     const reply = await req.object_sdk.delete_multiple_objects({

diff --git a/src/endpoint/s3/s3_utils.js b/src/endpoint/s3/s3_utils.js
@@ -227,9 +227,9 @@ function parse_sse(req) {
         }
     }
 
-    if (algorithm !== 'AES256' && algorithm !== 'aws:kms') throw new S3Error(S3Error.InvalidDigest);
+    if (algorithm !== 'AES256' && algorithm !== 'aws:kms') throw new S3Error(S3Error.InvalidArgument);
 
-    if (algorithm === 'aws:kms' && !kms_key_id) throw new S3Error(S3Error.InvalidDigest);
+    if (algorithm === 'aws:kms' && !kms_key_id) throw new S3Error(S3Error.InvalidArgument);
     // const md5_regex = new RegExp('/^[a-f0-9]{32}$/');
     // if (kms_key_id && !md5_regex.test(kms_key_id)) throw new S3Error(S3Error.InvalidDigest);
 

diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt
@@ -71,6 +71,7 @@ s3tests_boto3.functional.test_headers.test_bucket_create_bad_date_invalid_aws2
 s3tests_boto3.functional.test_headers.test_bucket_create_bad_date_none_aws2
 s3tests_boto3.functional.test_headers.test_bucket_create_bad_date_empty_aws2
 s3tests_boto3.functional.test_headers.test_bucket_create_bad_date_before_epoch_aws2
+s3tests.functional.test_headers.test_object_acl_create_contentlength_none
 s3tests_boto3.functional.test_s3.test_bucket_listv2_delimiter_prefix
 s3tests_boto3.functional.test_s3.test_bucket_listv2_delimiter_prefix_underscore
 s3tests_boto3.functional.test_s3.test_bucket_list_delimiter_unreadable
@@ -163,6 +164,7 @@ s3tests_boto3.functional.test_s3.test_object_acl_canned_bucketownerread
 s3tests_boto3.functional.test_s3.test_object_acl_canned_authenticatedread
 s3tests_boto3.functional.test_s3.test_object_acl_canned_bucketownerfullcontrol
 s3tests_boto3.functional.test_s3.test_object_acl_full_control_verify_owner
+s3tests_boto3.functional.test_s3.test_object_acl_full_control_verify_attributes
 s3tests_boto3.functional.test_s3.test_object_acl
 s3tests_boto3.functional.test_s3.test_object_acl_write
 s3tests_boto3.functional.test_s3.test_object_acl_writeacp
@@ -178,7 +180,7 @@ s3tests_boto3.functional.test_s3.test_bucket_acl_no_grants
 s3tests_boto3.functional.test_s3.test_bucket_header_acl_grants
 s3tests_boto3.functional.test_s3.test_object_header_acl_grants
 s3tests_boto3.functional.test_s3.test_bucket_acl_grant_email
-s3tests_boto3.functional.test_s3.test_bucket_acl_grant_email_notexist
+s3tests_boto3.functional.test_s3.test_bucket_acl_grant_email_not_exist
 s3tests_boto3.functional.test_s3.test_bucket_acl_revoke_all
 s3tests_boto3.functional.test_s3.test_logging_toggle
 s3tests_boto3.functional.test_s3.test_access_bucket_private_object_private

diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt
@@ -9,13 +9,19 @@ s3tests_boto3.functional.test_s3.test_bucket_list_prefix_basic
 s3tests_boto3.functional.test_s3.test_bucket_list_prefix_delimiter_basic
 s3tests_boto3.functional.test_s3.test_account_usage
 s3tests_boto3.functional.test_s3.test_head_bucket_usage
-s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit
 s3tests_boto3.functional.test_s3.test_post_object_invalid_signature
 s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key
 s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition
 s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field
 s3tests_boto3.functional.test_s3.test_post_object_expired_policy
 s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value
+s3tests_boto3.functional.test_s3.test_lifecycle_get_no_id
+s3tests_boto3.functional.test_s3.test_object_lock_put_obj_lock_invalid_mode
+s3tests_boto3.functional.test_s3.test_object_lock_get_obj_retention_iso8601
+s3tests_boto3.functional.test_s3.test_object_lock_delete_object_with_retention_and_marker
+s3tests_boto3.functional.test_s3.test_multipart_upload_on_a_bucket_with_policy
+s3tests_boto3.functional.test_s3.test_multipart_upload_small
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key
 s3tests_boto3.functional.test_s3.test_bucket_create_exists
 s3tests_boto3.functional.test_s3.test_bucket_create_exists_nonowner
 s3tests_boto3.functional.test_s3.test_bucket_recreate_overwrite_acl
@@ -31,10 +37,10 @@ s3tests_boto3.functional.test_s3.test_lifecycle_set_date
 s3tests_boto3.functional.test_s3.test_lifecycle_expiration_header_put
 s3tests_boto3.functional.test_s3.test_lifecycle_expiration_header_head
 s3tests_boto3.functional.test_s3.test_lifecycle_expiration_header_tags_head
+s3tests_boto3.functional.test_s3.test_lifecycle_transition_set_invalid_date
 s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_c_s3
 s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_c_kms
 s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_s3_kms
-s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_bad_enc_kms
 s3tests_boto3.functional.test_s3.test_bucket_policy_put_obj_s3_noenc
 s3tests_boto3.functional.test_s3.test_bucket_policy_put_obj_s3_kms
 s3tests_boto3.functional.test_s3.test_bucket_policy_put_obj_kms_noenc

diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_s3.js b/src/test/system_tests/ceph_s3_tests/test_ceph_s3.js
@@ -28,7 +28,7 @@ let tests_list;
 
 const OUT_OF_SCOPE_TESTS = create_out_of_scope_tests_list() || [];
 //Regexp match will be tested per each entry
-const S3_CEPH_TEST_OUT_OF_SCOPE_REGEXP = new RegExp(`(${OUT_OF_SCOPE_TESTS.join(')|(')})`);
+const S3_CEPH_TEST_OUT_OF_SCOPE_REGEXP = new RegExp(`(${OUT_OF_SCOPE_TESTS.join('\\b)|(')}\\b)`);
 const S3_CEPH_TEST_STEMS_REGEXP = new RegExp(`(${S3_CEPH_TEST_STEMS.join(')|(')})`);
 
 async function main() {