Enhancements to code and documentation incl bug fixes.

accelerators/CICD/Branch-out-to-new-workspace/AzDO/Branch_out_workspace.yml

@@ -9,48 +9,53 @@ parameters:
 - name: source_workspace
   displayName: Enter source workspace
   type: string
-  default: 'Dev_WS_CICDSample_3'
+  default: ''
 
 - name: target_workspace
   displayName: Enter target workspace name
   type: string
-  default: 'Dev_WS_CICDSample_Clone5'
+  default: ''
 
 - name: copy_lakehouse_data
   displayName: Copy Lakehouse Data (enter True or False)
   type: string
-  default: 'True'
+  default: 'False'
 
 - name: copy_warehouse_data
-  displayName: Copy Lakehouse Data (enter True or False)
+  displayName: Copy Warehouse Data (enter True or False)
   type: string
   default: 'False'
 
 - name: create_lakehouse_shortcuts
   displayName: Create lakehouse shortcuts (only if copy lakehouse data set to False)
   type: string
-  default: 'False'
+  default: 'True'
 
 - name: developer_email
   displayName: Enter developer email
   type: string
-  default: '[email protected]'
+  default: ''
 
 - name: capacity_id
   displayName: Enter capacity ID of the new workspace
   type: string
-  default: 'B34D9528-0FF8-4E40-865D-8BA769F574BB'
+  default: ''
 
 
 - name: ado_branch
   displayName: Enter the source branch name
   type: string
   default: 'main'
 
+- name: ado_git_folder
+  displayName: Folder in the repo where the Fabric content is stored. Leave as / if content is stored in root. 
+  type: string
+  default: "/"
+
 - name: connections_from_to
-  displayName: Swap connections in pipelines using names or IDs in the format (from,to) format
+  displayName: Swap connections in pipelines using names or IDs in the format (from,to) format. Leave as () if no connections to swap.
   type: string
-  default: "('4498340c-27cf-4c6e-a025-00e5de6b0726','4498340c-27cf-4c6e-a025-00e5de6b0726')"
+  default: "()"
 
 variables:
 - group: Fabric_Deployment_Group_S
@@ -77,7 +82,7 @@ stages:
             inputs:
               scriptSource: 'filePath'
               scriptPath: 'scripts/BranchOut-Feature-Workspace-Automation.py'
-              arguments: '--ADO_ORG_NAME $(ADO_ORG_NAME) --ADO_REPO_NAME $(ADO_REPO_NAME) --ADO_PROJECT_NAME $(ADO_PROJECT_NAME) --ADO_NEW_BRANCH ${{ parameters.target_workspace}} --DEVELOPER ${{ parameters.developer_email }} --WORKSPACE_NAME ${{ parameters.target_workspace }} --CAPACITY_ID ${{ parameters.capacity_id }} --ADO_API_URL $(ADO_API_URL) --ADO_MAIN_BRANCH ${{ parameters.ado_branch }}  --TENANT_ID $(TENANT_ID) --FABRIC_TOKEN $(fabrictoken) --ADO_PAT_TOKEN $(azdopat) --CLIENT_ID $(azclientid) --USER_NAME $(username) --PASSWORD $(password)'
+              arguments: '--ADO_ORG_NAME $(ADO_ORG_NAME) --ADO_REPO_NAME $(ADO_REPO_NAME) --ADO_PROJECT_NAME $(ADO_PROJECT_NAME) --ADO_NEW_BRANCH ${{ parameters.target_workspace}} --DEVELOPER ${{ parameters.developer_email }} --WORKSPACE_NAME ${{ parameters.target_workspace }} --CAPACITY_ID ${{ parameters.capacity_id }} --ADO_API_URL $(ADO_API_URL) --ADO_MAIN_BRANCH ${{ parameters.ado_branch }} --ADO_GIT_FOLDER ${{ parameters.ado_git_folder }} --TENANT_ID $(TENANT_ID) --FABRIC_TOKEN $(fabrictoken) --ADO_PAT_TOKEN $(azdopat) --CLIENT_ID $(azclientid) --USER_NAME $(username) --PASSWORD $(password)'
               #failOnStderr: true 
             displayName: 'Run Branch-Out-To-New-Workspace Script'
 

accelerators/CICD/Branch-out-to-new-workspace/AzDO/scripts/BranchOut-Feature-Workspace-Automation.py

@@ -18,6 +18,7 @@
 DEVELOPER = ""
 ADO_MAIN_BRANCH = ""
 ADO_NEW_BRANCH = ""
+ADO_GIT_FOLDER = ""
 ADO_PROJECT_NAME = ""
 ADO_REPO_NAME = ""
 ADO_ORG_NAME = ""
@@ -187,9 +188,9 @@ def get_branch_object_id(project_name, repo_name, branch_name, token, token_type
         return None
 
 # Function to connect Azure DevOps branch to Fabric workspace
-def connect_branch_to_workspace(workspace_id, project_name, org_name, repo_name, branch_name, token):
+def connect_branch_to_workspace(workspace_id, project_name, org_name, repo_name, branch_name, git_folder, token):
     try:
-        logging.info(f"Conecting workspace {workspace_id} to feature branch {branch_name} is in progess..")
+        logging.info(f"Conecting workspace {workspace_id} to feature branch {branch_name} at folder {git_folder}..")
         headers = {"Authorization": f"Bearer {token}"}
         data = {
         "gitProviderDetails": {
@@ -198,7 +199,7 @@ def connect_branch_to_workspace(workspace_id, project_name, org_name, repo_name,
                 "gitProviderType": "AzureDevOps",
                 "repositoryName": repo_name,
                 "branchName": branch_name,
-                "directoryName": ""
+                "directoryName": git_folder.rstrip("/")
          }
         } 
         response = requests.post(f"{FABRIC_API_URL}/workspaces/{workspace_id}/git/connect", headers=headers, json=data)
@@ -233,7 +234,7 @@ def long_running_operation_polling(uri,retry_after,headers):
 def initialize_workspace_from_git(workspace_id,token):
 
     try:
-        logging.info(f"Connecting f{WORKSPACE_NAME} to feature branch {ADO_NEW_BRANCH} is in propress... ")
+        logging.info(f"Initializing {WORKSPACE_NAME} to feature branch {ADO_NEW_BRANCH} is in propress... ")
         headers = {"Authorization": f"Bearer {token}"}
         # Initialize the connection to the GIT repository
         gitinitializeurl = f"{FABRIC_API_URL}/workspaces/{workspace_id}/git/initializeConnection"
@@ -290,6 +291,7 @@ def set_main_parameters():
     global DEVELOPER
     global ADO_MAIN_BRANCH
     global ADO_NEW_BRANCH
+    global ADO_GIT_FOLDER
     global ADO_PROJECT_NAME
     global ADO_REPO_NAME
     global ADO_ORG_NAME
@@ -311,6 +313,7 @@ def set_main_parameters():
         parser.add_argument('--WORKSPACE_NAME',type=str, help= 'Name of the feature workspace to be created')
         parser.add_argument('--DEVELOPER',type=str, help= 'Developr UPN to be added to workspace as admin')
         parser.add_argument('--ADO_MAIN_BRANCH',type=str, help= 'Main development branch')
+        parser.add_argument('--ADO_GIT_FOLDER',type=str, help= 'Folder where Fabric content is stored')
         parser.add_argument('--ADO_NEW_BRANCH',type=str, help= 'New branch to be created')
         parser.add_argument('--ADO_PROJECT_NAME',type=str, help= 'ADO project name')
         parser.add_argument('--ADO_REPO_NAME',type=str, help= 'ADO repository name')
@@ -324,6 +327,7 @@ def set_main_parameters():
         logging.error(f'Error: {e}')
         raise ValueError("Could not extract parameters: {e}")
 
+    logging.info('Binding parameters...')
     #Bind parameters to script variables
     TENANT_ID = args.TENANT_ID
     USERNAME = args.USER_NAME
@@ -332,6 +336,7 @@ def set_main_parameters():
     DEVELOPER = args.DEVELOPER
     ADO_MAIN_BRANCH = args.ADO_MAIN_BRANCH
     ADO_NEW_BRANCH = args.ADO_NEW_BRANCH
+    ADO_GIT_FOLDER = args.ADO_GIT_FOLDER
     ADO_PROJECT_NAME = args.ADO_PROJECT_NAME
     ADO_REPO_NAME = args.ADO_REPO_NAME
     ADO_ORG_NAME = args.ADO_ORG_NAME
@@ -364,10 +369,10 @@ def main():
            logging.info(f'Workspace {WORKSPACE_NAME} ({workspace_id}) successfully created and assigned to capacity {CAPACITY_ID}')
            logging.info(f'Adding workspace admins {DEVELOPER}...')
            add_workspace_admins(workspace_id, DEVELOPER, token)
-           logging.info(f'Creating ado branch from main {ADO_MAIN_BRANCH}...')
+           logging.info(f'Creating ado branch {ADO_NEW_BRANCH} from {ADO_MAIN_BRANCH}...')
            create_azure_devops_branch(ADO_PROJECT_NAME, ADO_REPO_NAME, ADO_MAIN_BRANCH, ADO_NEW_BRANCH)           
            logging.info(f'Connecting workspace to branch {ADO_NEW_BRANCH}...')
-           connect_branch_to_workspace(workspace_id, ADO_PROJECT_NAME, ADO_ORG_NAME,ADO_REPO_NAME, ADO_NEW_BRANCH, token)
+           connect_branch_to_workspace(workspace_id, ADO_PROJECT_NAME, ADO_ORG_NAME,ADO_REPO_NAME, ADO_NEW_BRANCH, ADO_GIT_FOLDER, token)
            logging.info('Initialize workspace...')
            initialize_workspace_from_git(workspace_id, token)
         else:

accelerators/CICD/Branch-out-to-new-workspace/AzDO/scripts/Run_post_activity.py

@@ -69,41 +69,54 @@ def acquire_token_user_id_password(tenant_id, client_id,user_name,password):
      logging.error('Error: Token could not be obtained: '+str(result))
    return access_token
 
-logging.info('Checking for supplied credentials...')
-if FABRIC_TOKEN!="":
-    logging.info('Fabric token found...')
-    token = FABRIC_TOKEN
-else:
-    logging.info('User creds found, generating token...')
-    token = acquire_token_user_id_password(TENANT_ID,CLIENT_ID,user_name,password)
-
-if token:
-    if NOTEBOOK_ID == '':
-        raise ValueError('Error: Could not execute notebook as no Notebook ID has been specified.')
+def main():
+    logging.info('Checking for supplied credentials...')
+    if FABRIC_TOKEN!="":
+        logging.info('Fabric token found...')
+        token = FABRIC_TOKEN
+    else:
+        logging.info('User creds found, generating token...')
+        token = acquire_token_user_id_password(TENANT_ID,CLIENT_ID,user_name,password)
+
+    if token:
+        if NOTEBOOK_ID == '':
+            raise ValueError('Error: Could not execute notebook as no Notebook ID has been specified.')
+
 
+        plurl = 'https://api.fabric.microsoft.com/v1/workspaces/'+WS_ID +'/items/'+NOTEBOOK_ID+'/jobs/instances?jobType=RunNotebook'
 
-    plurl = 'https://api.fabric.microsoft.com/v1/workspaces/'+WS_ID +'/items/'+NOTEBOOK_ID+'/jobs/instances?jobType=RunNotebook'
+        headers = {
+        "Authorization": f"Bearer {token}", 
+        "Content-Type": "application/json"  # Set the content type based on your request
+        }
+        logging.info('Setting notebook parameters...')
+        payload_data = '{' \
+            '"executionData": {' \
+                '"parameters": {' \
+                    '"_inlineInstallationEnabled": {"value": "True", "type": "bool"},' \
+                    '"source_ws": {"value": "' + SOURCE_WS + '", "type": "string"},' \
+                    '"copy_lakehouse_data": {"value": "' + COPY_LH + '", "type": "bool"},' \
+                    '"create_lakehouse_shortcuts": {"value": "' + CREATE_SC + '", "type": "bool"},' \
+                    '"copy_warehouse_data": {"value": "' + COPY_WH + '", "type": "bool"},' \
+                    '"target_ws": {"value": "' + TARGET_WS + '", "type": "string"},' \
+                    '"p_connections_from_to": {"value": "' + CONNECTIONS_FROM_TO + '", "type": "string"}' \
+                    '}}}'
+        logging.info('Invoking Fabric notebook job...')
+        plresponse = requests.post(plurl, json=json.loads(payload_data), headers=headers)
+        #logging.info(str(plresponse.status_code) + ' - ' + plresponse.text)    
+        if plresponse.status_code==202:
+            logging.info('Job invoked. Please check the Fabric monitoring hub to review the status of the job.')
+        else:
+            logging.error('An error occurred when trying to invoke job: ' + str(plresponse.status_code) + ' - ' + plresponse.text)
+            raise ValueError("Error invoking Fabric notebook. Please review the debug logs.")
 
-    headers = {
-    "Authorization": f"Bearer {token}", 
-    "Content-Type": "application/json"  # Set the content type based on your request
-    }
-    logging.info('Setting notebook parameters...')
-    payload_data = '{' \
-        '"executionData": {' \
-            '"parameters": {' \
-                '"_inlineInstallationEnabled": {"value": "True", "type": "bool"},' \
-                '"source_ws": {"value": "' + SOURCE_WS + '", "type": "string"},' \
-                '"copy_lakehouse_data": {"value": "' + COPY_LH + '", "type": "bool"},' \
-                '"create_lakehouse_shortcuts": {"value": "' + CREATE_SC + '", "type": "bool"},' \
-                '"copy_warehouse_data": {"value": "' + COPY_WH + '", "type": "bool"},' \
-                '"target_ws": {"value": "' + TARGET_WS + '", "type": "string"},' \
-                '"p_connections_from_to": {"value": "' + CONNECTIONS_FROM_TO + '", "type": "string"}' \
-                '}}}'
-    logging.info('Invoking Fabric notebook job...')
-    plresponse = requests.post(plurl, json=json.loads(payload_data), headers=headers)
-    logging.info(str(plresponse.status_code) + ' - ' + plresponse.text)    
-else:
-    logging.error("Could not aquire token")
-    raise ValueError("Could not generate authentication token. Please review the debug logs.")
+    else:
+        logging.error("Could not aquire token")
+        raise ValueError("Could not generate authentication token. Please review the debug logs.")
+
 
+
+if __name__ == "__main__":
+    logging.info('Starting Run_post_activity script...')
+    main()
+

accelerators/CICD/Branch-out-to-new-workspace/README.md

@@ -157,7 +157,7 @@ authentication method has been chosen:
 
 11. This will display run-time parameters which can be modified as necessary:
 
-<div><img align=right src=../../media/pipeline_params.png width=500>
+<div><img align=right src=../../media/pipeline_params.png width=300>
 <p>&nbsp;</p><p>&nbsp;</p> 
 a. Source workspace: Name of the dev workspace
 </p><p> b. Target workspace: Name of the new workspace to be created which will also serve as the new branch name
@@ -167,8 +167,9 @@ a. Source workspace: Name of the dev workspace
 </p><p> f. Swap connections in pipelines: Specify connections to be replaced using format (from 1 ,to1),(from2,to2),...(fromN,toN) using either connection ID or name.
 </p><p> g. Enter Developer Email: Add the email address of the developer to be granted admin role on the new workspace 
 </p><p> h. Enter Capacity ID: Enter the capacity ID of the new workspace if different from the default GUID.
-</p><p> i. Enter the branch name: Enter the source branch name which the new branch will be created from.
-</p><p> j. Click Run and monitor the release pipeline in Azure Devops and also the progress of the post activity notebook in the Fabric monitoring hub.
+</p><p> i. Enter the source branch name: Enter the source branch name which the new branch will be created from.
+</p><p> j. Enter the Git folder where Fabric content is stored. Leave as / if content is stored in root. 
+</p><p> k. Click Run and monitor the release pipeline in Azure Devops and also the progress of the post activity notebook in the Fabric monitoring hub.
 </div><p>&nbsp;</p><p>&nbsp;</p>
 <p>&nbsp;</p>
 12. To debug and monitor the running YAML pipeline click on the “BranchOut” job