adding upload folder flag and changing name for clarity

pyproject.toml

@@ -26,7 +26,7 @@ find-scene = "sar_antarctica.nci.cli:find_scene_file"
 find-orbits = "sar_antarctica.nci.cli:find_orbits_for_scene"
 run-pyrosar-gamma-workflow = "sar_antarctica.nci.cli:run_pyrosar_gamma_workflow"
 submit-pyrosar-gamma-workflow = "sar_antarctica.nci.cli:submit_pyrosar_gamma_workflow"
-push-folder-to-s3 = "sar_antarctica.nci.cli:push_folder_to_s3"
+upload-files-in-folder-to-s3 = "sar_antarctica.nci.cli:upload_files_in_folder_to_s3"
 
 [tool.pytest.ini_options]
 testpaths = ["tests/*"]

sar_antarctica/nci/cli.py

@@ -181,13 +181,18 @@ def find_orbits_for_scene(scene: str):
 @click.argument('src_folder', type=click.Path(exists=True, file_okay=False))
 @click.argument('s3_bucket')
 @click.argument('s3_bucket_folder')
+@click.option('--upload-folder', 
+              default=False, 
+              is_flag=True, 
+              help="Upload the whole folder to specified s3_bucket_folder.")
 @click.option('--exclude-extensions', '-e', multiple=True, help="File extensions to exclude, e.g., '.txt', '.log'")
 @click.option('--exclude-files', '-f', multiple=True, help="Specific files to exclude, e.g., 'config.json'")
 @click.option('--region-name', default='ap-southeast-2', show_default=True, help="AWS region name")
-def push_folder_to_s3(
+def upload_files_in_folder_to_s3(
         src_folder : str,
         s3_bucket : str,
         s3_bucket_folder : str,
+        upload_folder : bool,
         exclude_extensions : list[str] = [],
         exclude_files : list[str] = [],
         region_name : str = 'ap-southeast-2',
@@ -196,6 +201,7 @@ def push_folder_to_s3(
         src_folder = src_folder,
         s3_bucket = s3_bucket,
         s3_bucket_folder = s3_bucket_folder, 
+        upload_folder = upload_folder,
         exclude_extensions = exclude_extensions,
         exclude_files = exclude_files,
         region_name = region_name,

sar_antarctica/nci/upload/push_folder_to_s3.py

@@ -8,6 +8,7 @@ def push_files_in_folder_to_s3(
         src_folder : str,
         s3_bucket : str,
         s3_bucket_folder : str,
+        upload_folder : bool = False,
         exclude_extensions : list[str] = [],
         exclude_files : list[str] = [],
         region_name : str = 'ap-southeast-2',
@@ -23,6 +24,11 @@ def push_files_in_folder_to_s3(
         S3 bucket to push to
     s3_bucket_folder : str
         Folder within bucket to push to
+    upload_folder : bool
+        upload the entire folder to the s3_bucket_folder. 
+        If; src_folder = my/local_folder/ & s3_bucket_folder = s3/s3_folder
+        when True, all files uploaded to -> s3/s3_folder/local_folder/...
+        when False, all files uploaded to -> s3/s3_folder/...
     exclude_extensions : list[str], optional
         List of file extensions to exclude, by default []
     exclude_files : list[str], optional
@@ -57,6 +63,10 @@ def push_files_in_folder_to_s3(
                         continue
                 local_path = Path(root) / Path(file)
                 relative_path = Path(os.path.relpath(local_path, src_folder))
-                s3_key = Path(os.path.join(s3_bucket_folder, relative_path).replace("\\", "/"))
+                if not upload_folder:
+                    s3_key = Path(os.path.join(s3_bucket_folder, relative_path).replace("\\", "/"))
+                else:
+                    folder = Path(src_folder).name 
+                    s3_key = Path(os.path.join(s3_bucket_folder, folder, relative_path).replace("\\", "/"))
                 S3_CLIENT.upload_file(str(local_path), str(s3_bucket), str(s3_key))
                 logging.info(f"Uploaded {local_path} to s3://{s3_bucket}/{s3_key}")