ci: Multi-tenancy for tests and garbage collection

Signed-off-by: Oliver Koenig <[email protected]>

Author: ko3n1g

.github/workflows/cicd-main.yml

@@ -270,10 +270,15 @@ jobs:
         - name: Checkout repository
           uses: actions/checkout@v4
         - run: |
+            mkdir -p /home/TestData/nlp/megatron_gpt/starcoder-ci-hf/${{ github.run_id }};
             python scripts/checkpoint_converters/convert_starcoder_hf_to_nemo.py \
             --input_name_or_path /home/TestData/nlp/megatron_gpt/starcoder-ci-hf \
-            --output_path /home/TestData/nlp/megatron_gpt/starcoder-ci-hf
-            rm -f /home/TestData/nlp/megatron_gpt/starcoder-ci-hf/megatron_starcoder_tp1_pp1.nemo
+            --output_path /home/TestData/nlp/megatron_gpt/starcoder-ci-hf/${{ github.run_id }}
+        - name: Cleanup
+          if: "always()"
+          run: |
+            rm -rf /home/TestData/nlp/megatron_gpt/starcoder-ci-hf/megatron_starcoder_tp1_pp1.nemo; 
+            rm -rf /home/TestData/nlp/megatron_gpt/starcoder-ci-hf/${{ github.run_id }}/
         - uses: "NVIDIA/NeMo/.github/actions/cancel-workflow@main"
           if: "failure()"