Merge pull request #7326 from nadavMiz/new-ceph-tests

Update Ceph S3 Version to Newest

.github/workflows/ceph-s3-tests.yaml

@@ -34,7 +34,7 @@ jobs:
          # Freeze the version of operator
          # to avoid a failed run due to code changes in the operator repo.
          # Need to update the commit once in a while
-          ref: f7358501716816250702d9a4c96f2526f98534e7
+          ref: ca12ff9e360220bb50bb9a2b645846e4b241fa39
 
       - name: Change settings for k8s and minikube
         run: |
@@ -70,10 +70,11 @@ jobs:
           set -x
           cd ./noobaa-core
           kubectl apply -f ./src/test/system_tests/ceph_s3_tests/test_ceph_s3_job.yml
-          kubectl wait --for=condition=complete job/noobaa-tests-s3 --timeout=30m || TIMEOUT=true
+          kubectl wait --for=condition=complete job/noobaa-tests-s3 --timeout=45m || TIMEOUT=true
           kubectl logs job/noobaa-tests-s3 --tail 10000 -f
           if kubectl logs job/noobaa-tests-s3 | grep -q "Ceph Test Failed:"; then
             echo "At least one test failed!"
+            kubectl get pods
             exit 1
           fi
           if [ ${TIMEOUT} ]; then

docs/dev_guide/ceph_s3_tests/ceph_s3_tests_pending_list_status.md

@@ -36,4 +36,5 @@ Attached a table with tests that where investigated and their status (this table
 | test_schema_definition                                    | Faulty Test     |                      | Same as test_alias_cyclic_refernce |
 | test_progress_expressions                                 | Faulty Test     | [508](https://github.com/ceph/s3-tests/issues/508)                    |                      |
 | test_lifecycle_set_date                                   | Faulty Test     | [510](https://github.com/ceph/s3-tests/issues/510)                    |                      |
-| test_lifecycle_transition_set_invalid_date                | Not Implemented |                      |    added because of the following PR [7270](https://github.com/noobaa/noobaa-core/pull/7270#discussion_r1175123422)   |
+| test_lifecycle_transition_set_invalid_date                | Not Implemented |                      |    added because of the following PR [7270](https://github.com/noobaa/noobaa-core/pull/7270#discussion_r1175123422)   |
+| All aws4 tests (e.g test_bucket_create_bad_amz_date_before_epoch_aws4)                                   | Faulty Test     | [520](https://github.com/ceph/s3-tests/issues/520)                    |                      |

src/deploy/NVA_build/Tests.Dockerfile

@@ -15,7 +15,8 @@ ENV TEST_CONTAINER true
 RUN dnf group install -y -q "Development Tools" && \
     dnf install -y -q --nogpgcheck vim \
     which python3-virtualenv python36-devel libevent-devel libffi-devel libxml2-devel libxslt-devel zlib-devel \ 
-    git && \
+    git  \
+    tox && \
     dnf clean all
 
 WORKDIR /root/node_modules/noobaa-core/
@@ -26,9 +27,11 @@ WORKDIR /root/node_modules/noobaa-core/
 #   Size: ~ 83.9 MB
 #
 ##############################################################
-RUN ./src/test/system_tests/ceph_s3_tests/test_ceph_s3_deploy.sh $(pwd)
-RUN cd ./src/test/system_tests/ceph_s3_tests/s3-tests/ && \
-    ./bootstrap
+RUN ./src/test/system_tests/ceph_s3_tests/test_ceph_s3_deploy.sh $(pwd) 
+# add group permissions to s3-tests directory (tox needs it in order to run)
+RUN cd ./src/test/system_tests/ceph_s3_tests/ && \
+    chgrp -R 0 s3-tests && \
+    chmod -R g=u s3-tests
 
 ##############################################################
 # Layers:

src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt

@@ -1,72 +1,116 @@
-s3tests_boto3.functional.test_s3.test_get_bucket_policy_status
-s3tests.functional.test_s3.test_atomic_dual_conditional_write_1mb
-s3tests_boto3.functional.test_s3.test_bucket_listv2_encoding_basic
-s3tests_boto3.functional.test_s3.test_bucket_list_encoding_basic
-s3tests_boto3.functional.test_s3.test_bucket_list_delimiter_none
-s3tests_boto3.functional.test_s3.test_bucket_listv2_delimiter_none
-s3tests_boto3.functional.test_s3.test_bucket_list_delimiter_not_skip_special
-s3tests_boto3.functional.test_s3.test_bucket_list_prefix_basic
-s3tests_boto3.functional.test_s3.test_bucket_list_prefix_delimiter_basic
-s3tests_boto3.functional.test_s3.test_account_usage
-s3tests_boto3.functional.test_s3.test_head_bucket_usage
-s3tests_boto3.functional.test_s3.test_post_object_invalid_signature
-s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key
-s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition
-s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field
-s3tests_boto3.functional.test_s3.test_post_object_expired_policy
-s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value
-s3tests_boto3.functional.test_s3.test_lifecycle_get_no_id
-s3tests_boto3.functional.test_s3.test_object_lock_put_obj_lock_invalid_mode
-s3tests_boto3.functional.test_s3.test_object_lock_get_obj_retention_iso8601
-s3tests_boto3.functional.test_s3.test_object_lock_delete_object_with_retention_and_marker
-s3tests_boto3.functional.test_s3.test_multipart_upload_on_a_bucket_with_policy
-s3tests_boto3.functional.test_s3.test_multipart_upload_small
-s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key
-s3tests_boto3.functional.test_s3.test_bucket_create_exists
-s3tests_boto3.functional.test_s3.test_bucket_create_exists_nonowner
-s3tests_boto3.functional.test_s3.test_bucket_recreate_overwrite_acl
-s3tests_boto3.functional.test_s3.test_bucket_recreate_new_acl
-s3tests_boto3.functional.test_s3.test_list_multipart_upload_owner
-s3tests_boto3.functional.test_s3.test_lifecycle_set
-s3tests_boto3.functional.test_s3.test_lifecycle_expiration
-s3tests_boto3.functional.test_s3.test_lifecycle_expiration_versioning_enabled
-s3tests_boto3.functional.test_s3.test_lifecycle_expiration_tags2
-s3tests_boto3.functional.test_s3.test_lifecycle_expiration_versioned_tags2
-s3tests_boto3.functional.test_s3.test_lifecycle_expiration_noncur_tags1
-s3tests_boto3.functional.test_s3.test_lifecycle_set_date
-s3tests_boto3.functional.test_s3.test_lifecycle_expiration_header_put
-s3tests_boto3.functional.test_s3.test_lifecycle_expiration_header_head
-s3tests_boto3.functional.test_s3.test_lifecycle_expiration_header_tags_head
-s3tests_boto3.functional.test_s3.test_lifecycle_transition_set_invalid_date
-s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_c_s3
-s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_c_kms
-s3tests_boto3.functional.test_s3.test_put_obj_enc_conflict_s3_kms
-s3tests_boto3.functional.test_s3.test_bucket_policy_put_obj_s3_noenc
-s3tests_boto3.functional.test_s3.test_bucket_policy_put_obj_s3_kms
-s3tests_boto3.functional.test_s3.test_bucket_policy_put_obj_kms_noenc
-s3tests_boto3.functional.test_s3.test_bucket_policy_put_obj_kms_s3
-s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention
-s3tests_boto3.functional.test_s3.test_object_lock_put_legal_hold
-s3tests_boto3.functional.test_s3.test_object_lock_get_legal_hold
-s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_with_bypass
-s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass
-s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance
-s3tests_boto3.functional.test_s3.test_object_read_unreadable
-s3tests_boto3.functional.test_s3.test_user_policy
-s3tests_boto3.functional.test_s3.test_get_public_acl_bucket_policy_status
-s3tests_boto3.functional.test_s3.test_get_authpublic_acl_bucket_policy_status
-s3tests_boto3.functional.test_s3.test_get_publicpolicy_acl_bucket_policy_status
-s3tests_boto3.functional.test_s3.test_get_nonpublicpolicy_acl_bucket_policy_status
-s3tests_boto3.functional.test_s3.test_get_nonpublicpolicy_deny_bucket_policy_status
-s3tests_boto3.functional.test_s3.test_get_default_public_block
-s3tests_boto3.functional.test_s3.test_put_public_block
-s3tests_boto3.functional.test_s3.test_block_public_put_bucket_acls
-s3tests_boto3.functional.test_s3.test_block_public_object_canned_acls
-s3tests_boto3.functional.test_s3.test_block_public_policy
-s3tests_boto3.functional.test_s3.test_ignore_public_acls
-s3tests_boto3.functional.test_s3.test_sse_s3_default_post_object_authenticated_request
-s3tests_boto3.functional.test_s3.test_sse_kms_default_post_object_authenticated_request
-s3tests_boto3.functional.test_s3select.test_generate_projection
-s3tests_boto3.functional.test_s3select.test_alias_cyclic_refernce
-s3tests_boto3.functional.test_s3select.test_schema_definition
-s3tests_boto3.functional.test_s3select.test_progress_expressions
+s3tests_boto3/functional/test_s3.py::test_get_bucket_policy_status
+s3tests/functional/test_s3.py::test_atomic_dual_conditional_write_1mb
+s3tests_boto3/functional/test_s3.py::test_bucket_listv2_encoding_basic
+s3tests_boto3/functional/test_s3.py::test_bucket_list_encoding_basic
+s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_none
+s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_none
+s3tests_boto3/functional/test_s3.py::test_bucket_list_delimiter_not_skip_special
+s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_basic
+s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic
+s3tests_boto3/functional/test_s3.py::test_account_usage
+s3tests_boto3/functional/test_s3.py::test_head_bucket_usage
+s3tests_boto3/functional/test_s3.py::test_post_object_invalid_signature
+s3tests_boto3/functional/test_s3.py::test_post_object_invalid_access_key
+s3tests_boto3/functional/test_s3.py::test_post_object_missing_policy_condition
+s3tests_boto3/functional/test_s3.py::test_post_object_request_missing_policy_specified_field
+s3tests_boto3/functional/test_s3.py::test_post_object_expired_policy
+s3tests_boto3/functional/test_s3.py::test_post_object_invalid_request_field_value
+s3tests_boto3/functional/test_s3.py::test_lifecycle_get_no_id
+s3tests_boto3/functional/test_s3.py::test_object_lock_put_obj_lock_invalid_mode
+s3tests_boto3/functional/test_s3.py::test_object_lock_get_obj_retention_iso8601
+s3tests_boto3/functional/test_s3.py::test_object_lock_delete_object_with_retention_and_marker
+s3tests_boto3/functional/test_s3.py::test_multipart_upload_on_a_bucket_with_policy
+s3tests_boto3/functional/test_s3.py::test_multipart_upload_small
+s3tests_boto3/functional/test_s3.py::test_post_object_authenticated_request_bad_access_key
+s3tests_boto3/functional/test_s3.py::test_bucket_create_exists
+s3tests_boto3/functional/test_s3.py::test_bucket_create_exists_nonowner
+s3tests_boto3/functional/test_s3.py::test_bucket_recreate_overwrite_acl
+s3tests_boto3/functional/test_s3.py::test_bucket_recreate_new_acl
+s3tests_boto3/functional/test_s3.py::test_list_multipart_upload_owner
+s3tests_boto3/functional/test_s3.py::test_lifecycle_set
+s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration
+s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_versioning_enabled
+s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_tags2
+s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_versioned_tags2
+s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_noncur_tags1
+s3tests_boto3/functional/test_s3.py::test_lifecycle_set_date
+s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_header_put
+s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_header_head
+s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_header_tags_head
+s3tests_boto3/functional/test_s3.py::test_lifecycle_transition_set_invalid_date
+s3tests_boto3/functional/test_s3.py::test_put_obj_enc_conflict_c_s3
+s3tests_boto3/functional/test_s3.py::test_put_obj_enc_conflict_c_kms
+s3tests_boto3/functional/test_s3.py::test_put_obj_enc_conflict_s3_kms
+s3tests_boto3/functional/test_s3.py::test_bucket_policy_put_obj_s3_noenc
+s3tests_boto3/functional/test_s3.py::test_bucket_policy_put_obj_s3_kms
+s3tests_boto3/functional/test_s3.py::test_bucket_policy_put_obj_kms_noenc
+s3tests_boto3/functional/test_s3.py::test_bucket_policy_put_obj_kms_s3
+s3tests_boto3/functional/test_s3.py::test_object_lock_multi_delete_object_with_retention
+s3tests_boto3/functional/test_s3.py::test_object_lock_put_legal_hold
+s3tests_boto3/functional/test_s3.py::test_object_lock_get_legal_hold
+s3tests_boto3/functional/test_s3.py::test_object_lock_changing_mode_from_governance_with_bypass
+s3tests_boto3/functional/test_s3.py::test_object_lock_changing_mode_from_governance_without_bypass
+s3tests_boto3/functional/test_s3.py::test_object_lock_changing_mode_from_compliance
+s3tests_boto3/functional/test_s3.py::test_object_read_unreadable
+s3tests_boto3/functional/test_s3.py::test_user_policy
+s3tests_boto3/functional/test_s3.py::test_get_public_acl_bucket_policy_status
+s3tests_boto3/functional/test_s3.py::test_get_authpublic_acl_bucket_policy_status
+s3tests_boto3/functional/test_s3.py::test_get_publicpolicy_acl_bucket_policy_status
+s3tests_boto3/functional/test_s3.py::test_get_nonpublicpolicy_acl_bucket_policy_status
+s3tests_boto3/functional/test_s3.py::test_get_nonpublicpolicy_deny_bucket_policy_status
+s3tests_boto3/functional/test_s3.py::test_get_default_public_block
+s3tests_boto3/functional/test_s3.py::test_put_public_block
+s3tests_boto3/functional/test_s3.py::test_block_public_put_bucket_acls
+s3tests_boto3/functional/test_s3.py::test_block_public_object_canned_acls
+s3tests_boto3/functional/test_s3.py::test_block_public_policy
+s3tests_boto3/functional/test_s3.py::test_ignore_public_acls
+s3tests_boto3/functional/test_s3.py::test_sse_s3_default_post_object_authenticated_request
+s3tests_boto3/functional/test_s3.py::test_sse_kms_default_post_object_authenticated_request
+s3tests_boto3/functional/test_s3select.py::test_generate_projection
+s3tests_boto3/functional/test_s3select.py::test_alias_cyclic_refernce
+s3tests_boto3/functional/test_s3select.py::test_schema_definition
+s3tests_boto3/functional/test_s3select.py::test_progress_expressions
+s3tests_boto3/functional/test_s3.py::test_object_write_with_chunked_transfer_encoding
+s3tests_boto3/functional/test_s3.py::test_versioning_concurrent_multi_object_delete
+s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_rgw_chunk_size_bug
+s3tests_boto3/functional/test_s3.py::test_get_object_torrent
+s3tests_boto3/functional/test_s3select.py::test_count_json_operation
+s3tests_boto3/functional/test_s3select.py::test_column_sum_min_max
+s3tests_boto3/functional/test_s3.py::test_multipart_upload_resend_part
+s3tests/functional/test_headers.py::test_object_create_bad_md5_invalid_garbage_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_contentlength_mismatch_below_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_authorization_incorrect_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_authorization_invalid_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_ua_empty_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_ua_none_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_date_invalid_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_amz_date_invalid_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_date_empty_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_amz_date_empty_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_date_none_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_amz_date_none_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_date_before_today_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_amz_date_before_today_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_date_after_today_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_amz_date_after_today_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_date_before_epoch_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_amz_date_before_epoch_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_date_after_end_aws4
+s3tests/functional/test_headers.py::test_object_create_bad_amz_date_after_end_aws4
+s3tests/functional/test_headers.py::test_object_create_missing_signed_custom_header_aws4
+s3tests/functional/test_headers.py::test_object_create_missing_signed_header_aws4
+s3tests/functional/test_headers.py::test_bucket_create_bad_authorization_invalid_aws4
+s3tests/functional/test_headers.py::test_bucket_create_bad_ua_empty_aws4
+s3tests/functional/test_headers.py::test_bucket_create_bad_ua_none_aws4
+s3tests/functional/test_headers.py::test_bucket_create_bad_date_invalid_aws4
+s3tests/functional/test_headers.py::test_bucket_create_bad_amz_date_invalid_aws4
+s3tests/functional/test_headers.py::test_bucket_create_bad_date_empty_aws4
+s3tests/functional/test_headers.py::test_bucket_create_bad_amz_date_empty_aws4
+s3tests/functional/test_headers.py::test_bucket_create_bad_date_none_aws4
+s3tests/functional/test_headers.py::test_bucket_create_bad_amz_date_none_aws4
+s3tests/functional/test_headers.py::test_bucket_create_bad_date_before_today_aws4
+s3tests/functional/test_headers.py::test_bucket_create_bad_amz_date_before_today_aws4
+s3tests/functional/test_headers.py::test_bucket_create_bad_date_after_today_aws4
+s3tests/functional/test_headers.py::test_bucket_create_bad_amz_date_after_today_aws4
+s3tests/functional/test_headers.py::test_bucket_create_bad_date_before_epoch_aws4
+s3tests/functional/test_headers.py::test_bucket_create_bad_amz_date_before_epoch_aws4