Merge pull request #1037 from hubmapconsortium/sunset666/directory_permissions_reorganization

Adding permissions reset to reorganized datasets

Author: sunset666

src/ingest-pipeline/airflow/dags/reorganize_upload.py

@@ -15,6 +15,7 @@
 from airflow.operators.bash import BashOperator
 from airflow.exceptions import AirflowException
 from airflow.providers.http.hooks.http import HttpHook
+from airflow.decorators import task
 
 from hubmap_operators.common_operators import (
     LogInfoOperator,
@@ -38,6 +39,8 @@
 )
 
 from misc.tools.split_and_create import reorganize
+from misc.tools.set_standard_protections import process_one_uuid
+from misc.tools.survey import EntityFactory
 
 
 # Following are defaults which can be overridden later on
@@ -299,6 +302,21 @@ def my_callable(**kwargs):
     def xcom_consistency_puller(**kwargs):
         return kwargs["ti"].xcom_pull(task_ids="split_stage_2", key="child_uuid_list")
 
+
+    @task(task_id="permission_resetting")
+    def permission_resetting(**kwargs):
+        return_error = []
+        entity_host = HttpHook.get_connection("entity_api_connection").host
+        entity_factory = EntityFactory(get_auth_tok(**kwargs),
+                                       instance=find_matching_endpoint(entity_host))
+        for uuid in kwargs["ti"].xcom_pull(task_ids="split_stage_2", key="child_uuid_list"):
+            return_error.append(process_one_uuid(uuid, entity_factory))
+        if False in return_error:
+            return 1
+        return 0
+
+    t_reset_permissions = permission_resetting()
+
     t_md_consistency_tests = PythonOperator(
         task_id="md_consistency_tests",
         python_callable=utils.pythonop_md_consistency_tests,
@@ -414,6 +432,7 @@ def _get_upload_uuid(**kwargs):
         >> t_maybe_keep_2
         >> t_run_md_extract
         >> t_md_consistency_tests
+        >> t_reset_permissions
         >> t_send_status
         >> t_join
         >> t_preserve_info

src/ingest-pipeline/misc/tools/set_standard_protections.py

@@ -9,7 +9,10 @@
 from io import StringIO
 from typing import List
 
-from survey import Dataset, EntityFactory, is_uuid
+try:
+    from survey import Dataset, EntityFactory, is_uuid
+except ImportError:
+    from .survey import Dataset, EntityFactory, is_uuid
 
 
 logging.basicConfig()
@@ -33,7 +36,7 @@ def run_cmd(cmd: List[str]) -> int :
     return command.returncode
 
 
-def process_one_uuid(uuid: str, entity_factory: EntityFactory, **kwargs) -> bool :
+def process_one_uuid(uuid: str, entity_factory: EntityFactory, **kwargs) -> bool:
     LOGGER.info('handling uuid %s', uuid)
     try:
         ds = entity_factory.get(uuid)
@@ -64,7 +67,7 @@ def process_one_uuid(uuid: str, entity_factory: EntityFactory, **kwargs) -> bool
     acl_path = Path(__file__).absolute().parent.parent.parent / 'submodules'
     acl_path = acl_path / 'manual-data-ingest' / 'acl-settings' / acl_fname
     LOGGER.info('will apply %s', acl_path)
-    cmd1 = ['setfacl', '-b', str(ds.full_path)]
+    cmd1 = ['setfacl', '-bR', str(ds.full_path)]
     cmd2 = ['setfacl', '-R', '-M', str(acl_path), str(ds.full_path)]
     if kwargs.get('dry_run', False):
         cmd1.insert(1, '--test')