diff --git a/accelerators/CICD/Branch-out-to-new-workspace/Fabric/Branch out to new workspace - Post Activity.ipynb b/accelerators/CICD/Branch-out-to-new-workspace/Fabric/Branch out to new workspace - Post Activity.ipynb
index df71ce9..9fdd1eb 100644
--- a/accelerators/CICD/Branch-out-to-new-workspace/Fabric/Branch out to new workspace - Post Activity.ipynb	
+++ b/accelerators/CICD/Branch-out-to-new-workspace/Fabric/Branch out to new workspace - Post Activity.ipynb	
@@ -1 +1 @@
-{"cells":[{"cell_type":"code","source":["%pip -q install semantic-link-labs\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":true},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"1b03316d-c088-4a0e-a2f0-44d45d112121"},{"cell_type":"code","source":["%pip install jmespath"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"c68be6ba-7648-457f-af82-f1987d12d7f7"},{"cell_type":"code","source":["source_ws = ''\n","target_ws = ''\n","\n","# Either copy lakehouse data or create shortcuts, set at most one of these to True \n","copy_lakehouse_data = True\n","copy_warehouse_data = True\n","create_lakehouse_shortcuts = False\n","\n","\n","# If false then shortcuts will be created. If you wish to create shortcuts based on a pattern match please set the param below\n","# enter pattern match for creating shortcuts - see https://github.com/arasdk/fabric-code-samples/blob/main/shortcuts/fabric_shortcut_creator.py \n","PATTERN_MATCH = [\"*\"]\n","_inlineInstallationEnabled = True\n","\n","\n","p_connections_from_to = ()#('https://api.fabric.microsoft.com/v1/workspaces/ admin','4498340c-27cf-4c6e-a025-00e5de6b0726'),('4498340c-27cf-4c6e-a025-00e5de6b0726','https://api.fabric.microsoft.com/v1/workspaces/ admin'),('https://api.fabric.microsoft.com/v1/workspaces/ admin','4498340c-27cf-4c6e-a025-00e5de6b0726')"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"tags":["parameters"]},"id":"90efaa4f-846d-4924-900e-258837a3467d"},{"cell_type":"markdown","source":["##### Branch out to new workspace notebook\n","\n","This notebook runs post activity tasks after [branch out to new workspace functionality](https://blog.fabric.microsoft.com/en-us/blog/introducing-new-branching-capabilities-in-fabric-git-integration).\n","\n","In addition to this:\n","<ul>\n","<li>Default lakehouses are updated to the corresponding \"target\" lakehouse</li>\n","<li>Creates shortcuts to tables or shortcuts in the source lakehouse </li>\n","<li>Sets lakehouse connections for semantic models to \"local\" lakehouse</li> \n","<li>Rebinds reports to \"local\" semantic models</li></ul>\n","\n","Requirements:\n","<ul>\n","<li>Requires Semantic Link Labs installed by pip install below.</li>\n","<li>Requires JmesPath library for data pipeline JSON manipulation i.e. connection swaps</ul>\n","\n","Limitations of current script:\n","\n","<ul>\n","<li>Does recreate item shares or external shortcuts\n","</li>\n","</ul>\n","\n","\n"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"a98b6d0a-7a36-4116-ab0d-aa70144eb737"},{"cell_type":"markdown","source":["##### Install semantic link labs to support advanced functionality\n","\n","https://semantic-link-labs.readthedocs.io/en/latest/index.html\n","https://github.com/microsoft/semantic-link-labs/blob/main/README.md\n","\n"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"3b887bd6-a9c9-430f-b58f-b58a93f5ce29"},{"cell_type":"markdown","source":["##### Install Jmespath to make data pipeline changes such as updating linked notebooks, warehouses and lakehouses "],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"8a74ed11-dd64-43bb-a735-906a947c8666"},{"cell_type":"markdown","source":["##### Set parameters\n","Before running this notebook ensure these parameters are set correctly. If necessary these can be passed in via a data factory pipeline"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"dee81614-b92b-4242-890a-b11f97b1a640"},{"cell_type":"markdown","source":["##### Library imports and fabric rest client setup\n","\n","https://learn.microsoft.com/en-us/python/api/semantic-link-sempy/sempy.fabric.fabricrestclient"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"4fb01e1d-ec4e-4c69-b544-66f6d8c5a475"},{"cell_type":"code","source":["import pandas as pd\n","import datetime, time\n","import re,json, fnmatch,os\n","import requests, base64\n","import sempy\n","import sempy.fabric as fabric\n","from sempy.fabric.exceptions import FabricHTTPException, WorkspaceNotFoundException\n","from pyspark.sql import DataFrame\n","from pyspark.sql.functions import col,current_timestamp,lit\n","import sempy_labs as labs\n","from sempy_labs import migration, directlake\n","from sempy_labs import lakehouse as lake\n","from sempy_labs import report as rep\n","from sempy_labs.tom import connect_semantic_model\n","\n","# instantiate the Fabric rest client\n","client = fabric.FabricRestClient()\n","\n","# get the current workspace ID based on the context of where this notebook is run from\n","thisWsId = notebookutils.runtime.context['currentWorkspaceId']\n","thisWsName = notebookutils.runtime.context['currentWorkspaceName']\n","\n","source_ws_id = fabric.resolve_workspace_id(source_ws)\n","target_ws_id = fabric.resolve_workspace_id(target_ws)\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"391624c1-b299-452d-9ebf-f32626d49970"},{"cell_type":"markdown","source":["##### Update default lakehouses for notebooks\n","\n","Update notebook dependencies based on but now supports T-SQL notebooks:\n","https://github.com/PowerBiDevCamp/FabConWorkshopSweden/blob/main/DemoFiles/GitUpdateWorkspace/updateWorkspaceDependencies_v1.ipynb\n"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"aaae8a08-588d-4dd8-9d2c-2200b7a88d30"},{"cell_type":"code","source":["for notebook in notebookutils.notebook.list(workspaceId=target_ws_id):\n","    updates = False\n","    if notebook.displayName == 'ETL':#True: #notebook.displayName == 'T-SQL_Notebook': #notebook.displayName != 'Create Feature Branch':\n","\n","        # Get the current notebook definition\n","        json_payload = json.loads(notebookutils.notebook.getDefinition(notebook.displayName,workspaceId=source_ws_id))\n","        #print(json.dumps(json_payload, indent=4))\n","        # Check for any attached lakehouses\n","        if 'dependencies' in json_payload['metadata'] \\\n","            and 'lakehouse' in json_payload['metadata']['dependencies'] \\\n","            and json_payload['metadata'][\"dependencies\"][\"lakehouse\"] is not None:\n","            # Extract attached and default lakehouses\n","            current_lakehouse = json_payload['metadata']['dependencies']['lakehouse']\n","            # if default lakehouse setting exists\n","            if 'default_lakehouse_name' in current_lakehouse:\n","                #json_payload['metadata']['dependencies']['lakehouse'] = {}\n","                print(f\"Updating notebook {notebook.displayName} with new default lakehouse: {current_lakehouse['default_lakehouse_name']} in workspace {target_ws}\")\n","                source_lh_name = fabric.resolve_item_name(item_id = current_lakehouse['default_lakehouse'],type='Lakehouse',workspace=source_ws_id)\n","                #print(f\"source lh name {source_lh_name}\")\n","                current_lakehouse['default_lakehouse'] = fabric.resolve_item_id(item_name = source_lh_name,type='Lakehouse',workspace=target_ws_id)\n","                #current_lakehouse['default_lakehouse_name'] = \n","                current_lakehouse['default_lakehouse_workspace_id'] = target_ws_id\n","                updates = True\n","            # loop through all attached lakehouess\n","            for lakehouse in json_payload['metadata']['dependencies']['lakehouse']['known_lakehouses']:\n","                source_lh_id = lakehouse['id']\n","                # find source lakehouse name\n","                source_lh_name = fabric.resolve_item_name(item_id = lakehouse['id'],type='Lakehouse',workspace=source_ws_id)\n","                # find target lakehouse id based on name\n","                target_lh_id = fabric.resolve_item_id(item_name = source_lh_name,type='Lakehouse',workspace=target_ws_id)\n","                lakehouse['id'] = target_lh_id\n","                print(f'Updating attached lakehouse {source_lh_name} from {source_lh_id} to target ID {target_lh_id}')\n","                updates = True\n","\n","        if 'dependencies' in json_payload['metadata'] and 'warehouse' in json_payload['metadata']['dependencies']:\n","            # Fetch existing details\n","            current_warehouse = json_payload['metadata']['dependencies']['warehouse']\n","            current_warehouse_id = current_warehouse['default_warehouse']\n","            source_wh_name =  fabric.resolve_item_name(item_id = current_warehouse_id,workspace=source_ws_id)\n","            #print('Source warehouse name is ' + source_wh_name)\n","            target_wh_id = fabric.resolve_item_id(item_name = source_wh_name,type='Warehouse',workspace=target_ws_id)\n","\n","            if 'default_warehouse' in current_warehouse:\n","                #json_payload['metadata']['dependencies']['warehouse'] = {}\n","                print(f\"Attempting to update notebook {notebook.displayName} with new default warehouse: {target_wh_id} in {target_ws}\")\n","            \n","                json_payload['metadata']['dependencies']['warehouse']['default_warehouse'] = target_wh_id\n","                for warehouse in json_payload['metadata']['dependencies']['warehouse']['known_warehouses']:\n","                    if warehouse['id'] == current_warehouse_id:\n","                        warehouse['id'] = target_wh_id\n","                        updates = True\n","\n","        if updates:\n","            notebookutils.notebook.updateDefinition(\n","                    name = notebook.displayName,\n","                    content  = json.dumps(json_payload),\n","                    workspaceId = target_ws_id\n","                    )\n","            \n","            print(f\"Updated notebook {notebook.displayName} in {target_ws}\")\n","\n","        else:\n","            print(f'No default lakehouse set for notebook {notebook.displayName}, ignoring.')"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"5c60b5d2-f83c-46f8-9870-9fd609166b67"},{"cell_type":"markdown","source":["##### Run the below cell - contains utility functions to support lakehouse and warehouse initialisation\n","\n","Shortcut creator:\n","https://github.com/arasdk/fabric-code-samples/blob/main/shortcuts/fabric_shortcut_creator.py "],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"a47a56df-219d-4d6c-b950-491909638deb"},{"cell_type":"code","source":["##### \n","### Shortcut utility function \n","####\n","\n","# Extract workspace_id, item_id and path from a onelake URI\n","def extract_onelake_https_uri_components(uri):\n","    # Define a regular expression to match any string between slashes and capture the final path element(s) without the leading slash\n","    pattern = re.compile(r\"abfss://([^@]+)@[^/]+/([^/]+)/(.*)\")\n","    match = pattern.search(uri)\n","    if match:\n","        workspace_id, item_id, path = match.groups()\n","        return workspace_id, item_id, path\n","    else:\n","        return None, None, None\n","\n","\n","def is_valid_onelake_uri(uri: str) -> bool:\n","    workspace_id, item_id, path = extract_onelake_https_uri_components(uri)\n","    if \"abfss://\" not in uri or workspace_id is None or item_id is None or path is None:\n","        return False\n","\n","    return True\n","\n","\n","def get_last_path_segment(uri: str):\n","    path = uri.split(\"/\")  # Split the entire URI by '/'\n","    return path[-1] if path else None\n","\n","\n","def is_delta_table(uri: str):\n","    delta_log_path = os.path.join(uri, \"_delta_log\")\n","    return mssparkutils.fs.exists(delta_log_path)\n","\n","\n","def get_onelake_shorcut(workspace_id: str, item_id: str, path: str, name: str):\n","    shortcut_uri = (\n","        f\"v1/workspaces/{workspace_id}/items/{item_id}/shortcuts/{path}/{name}\"\n","    )\n","    result = client.get(shortcut_uri).json()\n","    return result\n","\n","\n","def is_folder_matching_pattern(path: str, folder_name: str, patterns: []):\n","    if folder_name in patterns:\n","        return True\n","    else:\n","        for pattern in patterns:\n","            if fnmatch.fnmatch(folder_name, pattern):\n","                return is_delta_table(path)\n","\n","    return False\n","\n","\n","def get_matching_delta_tables_uris(uri: str, patterns: []) -> []:\n","    # Use a set to avoid duplicates\n","    matched_uris = set()\n","    files = mssparkutils.fs.ls(uri)\n","    folders = [item for item in files if item.isDir]\n","\n","    # Filter folders to only those that matches the pattern and is a delta table\n","    matched_uris.update(\n","        folder.path\n","        for folder in folders\n","        if is_folder_matching_pattern(folder.path, folder.name, patterns)\n","    )\n","\n","    return matched_uris\n","\n","\n","def create_onelake_shorcut(source_uri: str, dest_uri: str):\n","    src_workspace_id, src_item_id, src_path = extract_onelake_https_uri_components(\n","        source_uri\n","    )\n","\n","    dest_workspace_id, dest_item_id, dest_path = extract_onelake_https_uri_components(\n","        dest_uri\n","    )\n","\n","    name = get_last_path_segment(source_uri)\n","    dest_uri_joined = os.path.join(dest_uri, name)\n","\n","    # If the destination path already exists, return without creating shortcut\n","    if mssparkutils.fs.exists(dest_uri_joined):\n","        print(f\"Destination already exists: {dest_uri_joined}\")\n","        return None\n","\n","    request_body = {\n","        \"name\": name,\n","        \"path\": dest_path,\n","        \"target\": {\n","            \"oneLake\": {\n","                \"itemId\": src_item_id,\n","                \"path\": src_path,\n","                \"workspaceId\": src_workspace_id,\n","            }\n","        },\n","    }\n","\n","    shortcut_uri = f\"v1/workspaces/{dest_workspace_id}/items/{dest_item_id}/shortcuts\"\n","    print(f\"Creating shortcut: {shortcut_uri}/{name}..\")\n","    try:\n","        client.post(shortcut_uri, json=request_body)\n","    except FabricHTTPException as e:\n","        print(e)\n","        return None\n","\n","    return get_onelake_shorcut(dest_workspace_id, dest_item_id, dest_path, name)\n","   \n","\n","####\n","## Copy lakehouse and warehouse utility functions\n","####\n","\n","def get_lh_object_list(base_path,data_types = ['Tables', 'Files'])->pd.DataFrame:\n","\n","    '''\n","    Function to get a list of tables for a lakehouse\n","    adapted from https://fabric.guru/getting-a-list-of-folders-and-delta-tables-in-the-fabric-lakehouse\n","    This function will return a pandas dataframe containing names and abfss paths of each folder for Files and Tables\n","    '''\n","    #data_types = ['Tables', 'Files'] #for if you want a list of files and tables\n","    #data_types = ['Tables'] #for if you want a list of tables\n","\n","    df = pd.concat([\n","        pd.DataFrame({\n","            'name': [item.name for item in notebookutils.fs.ls(f'{base_path}/{data_type}/')],\n","            'type': data_type[:-1].lower() , \n","            'src_path': [item.path for item in notebookutils.fs.ls(f'{base_path}/{data_type}/')],\n","        }) for data_type in data_types], ignore_index=True)\n","\n","    return df\n","\n","def get_wh_object_list(schema_list,base_path)->pd.DataFrame:\n","\n","    '''\n","    Function to get a list of tables for a warehouse by schema\n","    '''\n","    data_type = 'Tables'\n","    dfs = []\n","\n","    for schema_prefix in schema_list:\n","        if notebookutils.fs.exists(f'{base_path}/{data_type}/{schema_prefix}/'):\n","            items = notebookutils.fs.ls(f'{base_path}/{data_type}/{schema_prefix}/')\n","            if items:  # Check if the list is not empty\n","                df = pd.DataFrame({\n","                    'schema': schema_prefix,\n","                    'name': [item.name for item in items],\n","                    'type': data_type[:-1].lower(),\n","                    'src_path': [item.path for item in items],\n","                })\n","                dfs.append(df)\n","\n","    if dfs:  # Check if the list of dataframes is not empty\n","        df = pd.concat(dfs, ignore_index=True)\n","    else:\n","        df = pd.DataFrame()  # Return an empty dataframe if no dataframes were created\n","\n","    return df\n","\n","def copy_lh_objects(table_list,workspace_src,workspace_tgt,lakehouse_src,lakehouse_tgt,fastcopy=True,usingIDs=False)->pd.DataFrame:\n","    # declare an array to keep the instrumentation\n","    cpresult = []\n","    # loop through all the tables to extract the source path \n","    for table in table_list.src_path:\n","        source = table\n","        destination = source.replace(f'abfss://{workspace_src}', f'abfss://{workspace_tgt}')\n","        if usingIDs:\n","            destination = destination.replace(f'{lakehouse_src}', f'{lakehouse_tgt}')\n","        else:\n","            destination = destination.replace(f'{lakehouse_src}.Lakehouse', f'{lakehouse_tgt}.Lakehouse')\n","        start_time =  datetime.datetime.now()\n","        if notebookutils.fs.exists(destination):\n","             notebookutils.fs.rm(destination, True)\n","        if fastcopy:\n","            # use fastcopy util which is a python wrapper to azcopy\n","            notebookutils.fs.fastcp(source+'/*', destination+'/', True)\n","        else:\n","            notebookutils.fs.cp(source, destination, True)\n","\n","        # recording the timing and add it to the results list\n","        end_time = datetime.datetime.now()\n","        copyreslist = [source, destination, start_time.strftime(\"%Y-%m-%d %H:%M:%S\"),  end_time.strftime(\"%Y-%m-%d %H:%M:%S\"), str((end_time - start_time).total_seconds())]\n","        cpresult.append(copyreslist)\n","    return pd.DataFrame(cpresult,columns =['source--------------------------------------','target--------------------------------------','start------------','end_time------------','elapsed seconds----'])\n","\n","def createDWrecoverypl(ws_id,pl_name = 'Recover_Warehouse_Data_From_DR'):\n","  client = fabric.FabricRestClient()\n","\n","  dfurl= \"v1/workspaces/\"+ ws_id + \"/items\"\n","  payload = { \n","  \"displayName\": pl_name, \n","  \"type\": \"DataPipeline\", \n","  \"definition\": { \n","    \"parts\": [ \n","      { \n","        \"path\": \"pipeline-content.json\", \n","        \"payload\":  \"ewogICAgInByb3BlcnRpZXMiOiB7CiAgICAgICAgImFjdGl2aXRpZXMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICJuYW1lIjogIkl0ZXJhdGVTY2hlbWFUYWJsZXMiLAogICAgICAgICAgICAgICAgInR5cGUiOiAiRm9yRWFjaCIsCiAgICAgICAgICAgICAgICAiZGVwZW5kc09uIjogW10sCiAgICAgICAgICAgICAgICAidHlwZVByb3BlcnRpZXMiOiB7CiAgICAgICAgICAgICAgICAgICAgIml0ZW1zIjogewogICAgICAgICAgICAgICAgICAgICAgICAidmFsdWUiOiAiQHBpcGVsaW5lKCkucGFyYW1ldGVycy50YWJsZXNUb0NvcHkiLAogICAgICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJFeHByZXNzaW9uIgogICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgImJhdGNoQ291bnQiOiAyMCwKICAgICAgICAgICAgICAgICAgICAiYWN0aXZpdGllcyI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAgICAgIm5hbWUiOiAiQ29weVdhcmVob3VzZVRhYmxlcyIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJDb3B5IiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkZXBlbmRzT24iOiBbCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYWN0aXZpdHkiOiAiU2V0IHRhYmxlIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImRlcGVuZGVuY3lDb25kaXRpb25zIjogWwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIlN1Y2NlZWRlZCIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgXQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAicG9saWN5IjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0aW1lb3V0IjogIjAuMTI6MDA6MDAiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJyZXRyeSI6IDIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInJldHJ5SW50ZXJ2YWxJblNlY29uZHMiOiAzMCwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAic2VjdXJlT3V0cHV0IjogZmFsc2UsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInNlY3VyZUlucHV0IjogZmFsc2UKICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZVByb3BlcnRpZXMiOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInNvdXJjZSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInR5cGUiOiAiRGF0YVdhcmVob3VzZVNvdXJjZSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJxdWVyeVRpbWVvdXQiOiAiMDI6MDA6MDAiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAicGFydGl0aW9uT3B0aW9uIjogIk5vbmUiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZGF0YXNldFNldHRpbmdzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImFubm90YXRpb25zIjogW10sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibGlua2VkU2VydmljZSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibmFtZSI6ICIwN2EwMzAwNl9kMWI2XzRhMzlfYmViMV8wYmJhMmFhZjVmZjciLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJwcm9wZXJ0aWVzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYW5ub3RhdGlvbnMiOiBbXSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInR5cGUiOiAiRGF0YVdhcmVob3VzZSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0eXBlUHJvcGVydGllcyI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlbmRwb2ludCI6ICJAcGlwZWxpbmUoKS5wYXJhbWV0ZXJzLmxha2Vob3VzZUNvbm5TdHIiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImFydGlmYWN0SWQiOiAiQHBpcGVsaW5lKCkucGFyYW1ldGVycy5sYWtlaG91c2VJZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAid29ya3NwYWNlSWQiOiAiQHBpcGVsaW5lKCkucGFyYW1ldGVycy53b3Jrc3BhY2VJZCIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJEYXRhV2FyZWhvdXNlVGFibGUiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInNjaGVtYSI6IFtdLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInR5cGVQcm9wZXJ0aWVzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzY2hlbWEiOiAiZGJvIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidGFibGUiOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ2YWx1ZSI6ICJAY29uY2F0KGNvbmNhdChpdGVtKCkuc2NoZW1hLCdfJyksaXRlbSgpLm5hbWUpIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInR5cGUiOiAiRXhwcmVzc2lvbiIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzaW5rIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJEYXRhV2FyZWhvdXNlU2luayIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJhbGxvd0NvcHlDb21tYW5kIjogdHJ1ZSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInRhYmxlT3B0aW9uIjogImF1dG9DcmVhdGUiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZGF0YXNldFNldHRpbmdzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImFubm90YXRpb25zIjogW10sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibGlua2VkU2VydmljZSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibmFtZSI6ICIwYzAzMTIzYV9kMzEyXzQ2YzRfYThlN181YjRjYWQ4ZjEyZDciLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJwcm9wZXJ0aWVzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYW5ub3RhdGlvbnMiOiBbXSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInR5cGUiOiAiRGF0YVdhcmVob3VzZSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0eXBlUHJvcGVydGllcyI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlbmRwb2ludCI6ICJAcGlwZWxpbmUoKS5wYXJhbWV0ZXJzLndhcmVob3VzZUNvbm5TdHIiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImFydGlmYWN0SWQiOiAiQHBpcGVsaW5lKCkucGFyYW1ldGVycy53YXJlaG91c2VJZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAid29ya3NwYWNlSWQiOiAiQHBpcGVsaW5lKCkucGFyYW1ldGVycy53b3Jrc3BhY2VJZCIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJEYXRhV2FyZWhvdXNlVGFibGUiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInNjaGVtYSI6IFtdLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInR5cGVQcm9wZXJ0aWVzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzY2hlbWEiOiAiZGJvIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidGFibGUiOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ2YWx1ZSI6ICJAaXRlbSgpLm5hbWUiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJFeHByZXNzaW9uIgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVuYWJsZVN0YWdpbmciOiB0cnVlLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0cmFuc2xhdG9yIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJUYWJ1bGFyVHJhbnNsYXRvciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0eXBlQ29udmVyc2lvbiI6IHRydWUsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0eXBlQ29udmVyc2lvblNldHRpbmdzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImFsbG93RGF0YVRydW5jYXRpb24iOiB0cnVlLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInRyZWF0Qm9vbGVhbkFzTnVtYmVyIjogZmFsc2UKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAgICAgIm5hbWUiOiAiU2V0IHRhYmxlIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0eXBlIjogIlNldFZhcmlhYmxlIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkZXBlbmRzT24iOiBbCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYWN0aXZpdHkiOiAiU2V0IHNjaGVtYSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkZXBlbmRlbmN5Q29uZGl0aW9ucyI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJTdWNjZWVkZWQiCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICAgICAgICAgICAgICBdLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgInBvbGljeSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAic2VjdXJlT3V0cHV0IjogZmFsc2UsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInNlY3VyZUlucHV0IjogZmFsc2UKICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZVByb3BlcnRpZXMiOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInZhcmlhYmxlTmFtZSI6ICJUYWJsZW5hbWUiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ2YWx1ZSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInZhbHVlIjogIkBpdGVtKCkubmFtZSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0eXBlIjogIkV4cHJlc3Npb24iCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAibmFtZSI6ICJTZXQgc2NoZW1hIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0eXBlIjogIlNldFZhcmlhYmxlIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkZXBlbmRzT24iOiBbXSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICJwb2xpY3kiOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInNlY3VyZU91dHB1dCI6IGZhbHNlLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzZWN1cmVJbnB1dCI6IGZhbHNlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgInR5cGVQcm9wZXJ0aWVzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ2YXJpYWJsZU5hbWUiOiAiU2NoZW1hbmFtZSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInZhbHVlIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidmFsdWUiOiAiQGl0ZW0oKS5zY2hlbWEiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJFeHByZXNzaW9uIgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgfQogICAgICAgIF0sCiAgICAgICAgInBhcmFtZXRlcnMiOiB7CiAgICAgICAgICAgICJsYWtlaG91c2VJZCI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogInN0cmluZyIsCiAgICAgICAgICAgICAgICAiZGVmYXVsdFZhbHVlIjogIjBmMGY2YjdjLTE3NjEtNDFlNi04OTZlLTMwMDE0ZjE2ZmY2ZCIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgInRhYmxlc1RvQ29weSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogImFycmF5IiwKICAgICAgICAgICAgICAgICJkZWZhdWx0VmFsdWUiOiBbCiAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAic2NoZW1hIjogImRibyIsCiAgICAgICAgICAgICAgICAgICAgICAgICJuYW1lIjogIkRhdGUiCiAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgICAgICJzY2hlbWEiOiAiZGJvIiwKICAgICAgICAgICAgICAgICAgICAgICAgIm5hbWUiOiAiR2VvZ3JhcGh5IgogICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAic2NoZW1hIjogImRibyIsCiAgICAgICAgICAgICAgICAgICAgICAgICJuYW1lIjogIkhhY2tuZXlMaWNlbnNlIgogICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAic2NoZW1hIjogImRibyIsCiAgICAgICAgICAgICAgICAgICAgICAgICJuYW1lIjogIk1lZGFsbGlvbiIKICAgICAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAgICAgInNjaGVtYSI6ICJkYm8iLAogICAgICAgICAgICAgICAgICAgICAgICAibmFtZSI6ICJUaW1lIgogICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAic2NoZW1hIjogImRibyIsCiAgICAgICAgICAgICAgICAgICAgICAgICJuYW1lIjogIlRyaXAiCiAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgICAgICJzY2hlbWEiOiAiZGJvIiwKICAgICAgICAgICAgICAgICAgICAgICAgIm5hbWUiOiAiV2VhdGhlciIKICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICBdCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgICJ3b3Jrc3BhY2VJZCI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogInN0cmluZyIsCiAgICAgICAgICAgICAgICAiZGVmYXVsdFZhbHVlIjogIjE1MDExNDNjLTI3MmYtNGEyZi05NzZhLTdlNTU5NzFlNGMyYiIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgIndhcmVob3VzZUlkIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAic3RyaW5nIiwKICAgICAgICAgICAgICAgICJkZWZhdWx0VmFsdWUiOiAiNGQxYmQ5NTEtOTlkZS00YmQ3LWI3YmMtNzFjOGY1NmRiNDExIgogICAgICAgICAgICB9LAogICAgICAgICAgICAid2FyZWhvdXNlQ29ublN0ciI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogInN0cmluZyIsCiAgICAgICAgICAgICAgICAiZGVmYXVsdFZhbHVlIjogIjcyd3diaXZpMnViZWpicnRtdGFobzMyYjR5LWhxa2FjZmpwZTR4dXZmM2twemt6b2hzbWZtLmRhdGF3YXJlaG91c2UuZmFicmljLm1pY3Jvc29mdC5jb20iCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgICJsYWtlaG91c2VDb25uU3RyIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAic3RyaW5nIiwKICAgICAgICAgICAgICAgICJkZWZhdWx0VmFsdWUiOiAiNzJ3d2JpdmkydWJlamJydG10YWhvMzJiNHktaHFrYWNmanBlNHh1dmYza3B6a3pvaHNtZm0uZGF0YXdhcmVob3VzZS5mYWJyaWMubWljcm9zb2Z0LmNvbSIKICAgICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgInZhcmlhYmxlcyI6IHsKICAgICAgICAgICAgIlRhYmxlbmFtZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIlN0cmluZyIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgIlNjaGVtYW5hbWUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJTdHJpbmciCiAgICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgICJsYXN0TW9kaWZpZWRCeU9iamVjdElkIjogIjRhYTIwYWY3LTk0YmQtNDM0OC1iZWY4LWY4Y2JjZDg0MGQ1MSIsCiAgICAgICAgImxhc3RQdWJsaXNoVGltZSI6ICIyMDI0LTExLTEzVDE1OjUyOjUyWiIKICAgIH0KfQ==\", \n","        \"payloadType\": \"InlineBase64\" \n","      } \n","    ] \n","  } \n","}   \n","  \n","  response = json.loads(client.post(dfurl,json= payload).content)\n","  return response['id']\n","\n","def getItemId(wks_id,itm_name,itm_type):\n","    df = fabric.list_items(type=None,workspace=wks_id)\n","    #print(df)\n","    if df.empty:\n","        return 'NotExists'\n","    else:\n","        #display(df)\n","        #print(df.query('\"Display Name\"=\"'+itm_name+'\"'))\n","        if itm_type != '':\n","            newdf= df.loc[(df['Display Name'] == itm_name) & (df['Type'] == itm_type)]['Id']\n","        else:\n","            newdf= df.loc[(df['Display Name'] == itm_name)]['Id']  \n","        if newdf.empty:\n","            return 'NotExists'\n","        else:\n","            return newdf.iloc[0]\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":true,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"collapsed":false},"id":"e46210e9-58c9-483a-84ae-bbdc2ad1c37f"},{"cell_type":"markdown","source":["##### Either create shortcuts from source to target lakehouse(s) or copy data\n","\n","Loops through lakehouse(s) in the target workspace and either populates them with shortcuts or data\n","\n"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"a15065f3-670d-4bc9-b337-51709f6cdb1f"},{"cell_type":"code","source":["df_lhs = labs.list_lakehouses(source_ws)\n","for index, row in df_lhs.iterrows():\n","\n","\n","    if copy_lakehouse_data:\n","        df_lakehouses = (labs.list_lakehouses(source_ws))\n","        lh_name= row['Lakehouse Name']\n","        if lh_name.find('temp')==-1:\n","            # Gathers the list of recovers tables and source paths to be copied into the lakehouse associated with this notebook \n","            src_path = f'abfss://{source_ws}@onelake.dfs.fabric.microsoft.com/{lh_name}.Lakehouse'\n","\n","            table_list = get_lh_object_list(src_path)\n","            print(f'Attempting to copy table data for lakehouse {lh_name} from workspace {source_ws} to {target_ws}...')\n","            display(table_list)\n","\n","            #print('Copy Lakehouse Delta tables...')\n","            res = copy_lh_objects(table_list[table_list['type']=='table'],source_ws,target_ws,\n","                                lh_name,lh_name,False,False)\n","            display(res)\n","            # Copy files\n","            print(f'Attempting to copy file data for lakehouse {lh_name} from workspace {source_ws} to {target_ws}...')\n","\n","            #print('Copy Lakehouse files...')\n","            res = copy_lh_objects(table_list[table_list['type']=='file'],source_ws,target_ws,\n","                                lh_name,lh_name,False,False)\n","            display(res)\n","            print('Done.')\n","\n","    else:\n","        # fetch ID of source lakehouse based on name and workspace\n","        source_lh_id = fabric.resolve_item_id(\n","            item_name=row['Lakehouse Name'], type=\"Lakehouse\", workspace=source_ws\n","            )\n","        #target_lh_id = notebookutils.lakehouse.getWithProperties(name=current_lakehouse['default_lakehouse_name'], workspaceId=new_workspace_id)['id']\n","\n","        SOURCE_URI = f\"abfss://{source_ws_id}@onelake.dfs.fabric.microsoft.com/{source_lh_id}/Tables\"\n","        DEST_URI = f\"abfss://{target_ws_id}@onelake.dfs.fabric.microsoft.com/{row['Lakehouse ID']}/Tables\"\n","\n","        if PATTERN_MATCH is None or len(PATTERN_MATCH) == 0:\n","            raise TypeError(\"Argument 'PATTERN_MATCH' should be a valid list of patterns or [\"*\"] to match everything\")\n","\n","        # Collect created shortcuts\n","        result = []\n","\n","        # If either URI's are invalid, just return\n","        if not is_valid_onelake_uri(SOURCE_URI) or not is_valid_onelake_uri(DEST_URI):\n","            print(\n","                \"invalid URI's provided. URI's should be in the form: abfss://<workspace-id>@onelake.dfs.fabric.microsoft.com/<item-id>/<path>\"\n","            )\n","        else:\n","            # Remove any trailing '/' from uri's\n","            source_uri_addr = SOURCE_URI.rstrip(\"/\")\n","            dest_uri_addr = DEST_URI.rstrip(\"/\")\n","\n","            dest_workspace_id, dest_item_id, dest_path = extract_onelake_https_uri_components(\n","                dest_uri_addr\n","            )\n","\n","            # If we are not shortcutting to a managed table folder or\n","            # the source uri is a delta table, just shortcut it 1-1.\n","            if not dest_path.startswith(\"Tables\") or is_delta_table(source_uri_addr):\n","                shortcut = create_onelake_shorcut(source_uri_addr, dest_uri_addr)\n","                if shortcut is not None:\n","                    result.append(shortcut)\n","            else:\n","                # If source is not a delta table, and destination is managed table folder:\n","                # Iterate over source folders and create table shortcuts @ destination\n","                for delta_table_uri in get_matching_delta_tables_uris(\n","                    source_uri_addr, PATTERN_MATCH\n","                ):\n","                    shortcut = create_onelake_shorcut(delta_table_uri, dest_uri_addr)\n","                    if shortcut is not None:\n","                        result.append(shortcut)\n","        print(result)\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"collapsed":false},"id":"8bec3fbc-4a75-4ba3-86d5-0620ec504a8f"},{"cell_type":"markdown","source":["##### Copy warehouse data via parameterised pipeline\n","\n","Loop through all warehouses and copy the data"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"f198b816-77e9-4f04-9139-d78237bedc72"},{"cell_type":"code","source":["p_logging_verbose = True\n","df_warehouses = (labs.list_warehouses(target_ws))\n","display(df_warehouses)\n","for index, row in df_warehouses.iterrows():\n","    source_wh_id = labs.resolve_warehouse_id(row['Warehouse Name'],source_ws_id)\n","    target_wh_id = labs.resolve_warehouse_id(row['Warehouse Name'],target_ws_id)\n","    \n","    src_path = f'abfss://'+source_ws_id+'@onelake.dfs.fabric.microsoft.com/'+source_wh_id\n","    tgt_path = f'abfss://'+target_ws_id+'@onelake.dfs.fabric.microsoft.com/'+target_wh_id\n","\n","    # extract the list of schemas per data \n","    schema_list = get_lh_object_list(src_path,['Tables'])\n","    # extract a list of warehouse objects per schema and store in a list\n","    table_list = get_wh_object_list(schema_list['name'],src_path)\n","  \n","    # create a temporary staging lakehouse per warehouse to create shortcuts into, \n","    # which point back to original warehouse data currently in the DR storage account\n","    lhname = 'temp_rlh_' + source_ws+'_'+row['Warehouse Name']\n","    # check if it exists before attempting create\n","    if p_logging_verbose:\n","        print('Checking whether the temporary lakehouse \"'+ lhname +'\" exists in workspace '+target_ws+'...')\n","    temp_lh_id = getItemId(target_ws_id,lhname,'Lakehouse')\n","    if temp_lh_id == 'NotExists':\n","        lhname = lhname[:256] # lakehouse name should not exceed 256 characters\n","        payload = payload = '{\"displayName\": \"' + lhname + '\",' \\\n","        + '\"description\":  \"Interim staging lakehouse for primary warehouse recovery: ' \\\n","        + source_ws+'_'+row['Warehouse Name'] + 'into workspace '+ target_ws + '(' + target_ws +')\"}'\n","        try:\n","            lhurl = \"v1/workspaces/\" + target_ws_id + \"/lakehouses\"\n","            lhresponse = client.post(lhurl,json= json.loads(payload))\n","            temp_lh_id = lhresponse.json()['id']\n","            if p_logging_verbose:\n","                print('Temporary lakehouse \"'+ lhname +'\" created with Id ' + temp_lh_id + ': ' + str(lhresponse.status_code) + ' ' + str(lhresponse.text))\n","        except Exception as error:\n","            print(error.errorCode)\n","    else:\n","        if p_logging_verbose:\n","            print('Temporary lakehouse '+lhname+' (' + temp_lh_id + ') already exists.')\n","        \n","    time.sleep(60) # waiting for temporary lakehouse to provision completely   \n","\n","    # Create shortcuts for every table in the format of schema_table under the tables folder\n","    for index,itable in table_list.iterrows():\n","        shortcutExists=False\n","        # Check if shortcut exists\n","        try:\n","            url = \"v1/workspaces/\" + target_ws_id + \"/items/\" + temp_lh_id + \"/shortcuts/Tables/\"+itable['schema']+'_'+itable['name']\n","            tlhresponse = client.get(url)\n","            shortcutExists = True\n","            if p_logging_verbose:\n","                print('Shortcut '+itable['schema']+'_'+itable['name'] +' already exists')\n","        except Exception as error:\n","            shortcutExists = False    \n","\n","        if not shortcutExists: \n","            # Create shortcuts - one per table per schema\n","            url = \"v1/workspaces/\" + target_ws_id + \"/items/\" + temp_lh_id + \"/shortcuts\"\n","            scpayload = '{' \\\n","            '\"path\": \"Tables/\",' \\\n","            '\"name\": \"'+itable['schema']+'_'+itable['name']+'\",' \\\n","            '\"target\": {' \\\n","            '\"oneLake\": {' \\\n","                '\"workspaceId\": \"' + source_ws_id + '\",' \\\n","                '\"itemId\": \"'+ source_wh_id +'\",' \\\n","                '\"path\": \"/Tables/' + itable['schema']+'/'+itable['name'] + '\"' \\\n","                '}}}' \n","            try:\n","                #print(scpayload)                \n","                shctresponse = client.post(url,json= json.loads(scpayload))\n","                if p_logging_verbose:\n","                    print('Shortcut '+itable['schema']+'_'+itable['name'] + ' created.' )\n","\n","            except Exception as error:\n","                print('Error creating shortcut '+itable['schema']+'_'+itable['name']+' due to '+str(error) + ':' + shctresponse.text)\n","    \n","    recovery_pipeline_prefix= 'plRecover_WH'       \n","    # recovery pipeline name should not exceed 256 characters\n","    recovery_pipeline = recovery_pipeline_prefix+'_'+source_ws + '_'+row['Warehouse Name'][:256]\n","    if p_logging_verbose:\n","        print('Attempting to deploy a copy pipeline in the target workspace to load the target warehouse tables from the shortcuts created above... ')\n","    # Create the pipeline in the target workspace that loads the target warehouse from shortcuts created above \n","    plid = getItemId( target_ws_id,recovery_pipeline,'DataPipeline')\n","    #print(plid)\n","    if plid == 'NotExists':\n","      plid = createDWrecoverypl(target_ws_id,recovery_pipeline_prefix+'_'+source_ws + '_'+row['Warehouse Name'])\n","      if p_logging_verbose:\n","          print('Recovery pipeline ' + recovery_pipeline + ' created with Id '+plid)\n","    else:\n","      if p_logging_verbose:\n","          print('Datawarehouse recovery pipeline \"' + recovery_pipeline + '\" ('+plid+') already exist in workspace \"'+target_ws + '\" ('+target_ws_id+')')  \n","          print('\\n')\n","\n","    tablesToCopyParam = table_list[['schema','name']].to_json( orient='records')\n","    # ensure the temporary lakehouse exists\n","\n","    # obtain the connection string for the lakehouse to pass to the copy pipeline\n","    whurl  = \"v1/workspaces/\" + target_ws_id + \"/lakehouses/\" + temp_lh_id\n","    whresponse = client.get(whurl)\n","    lhconnStr = whresponse.json()['properties']['sqlEndpointProperties']['connectionString']\n","\n","    # get the SQLEndpoint ID of the lakehouse to pass to the copy pipeline\n","    items = fabric.list_items(workspace=target_ws_id)\n","    print(items)\n","    temp_lh_sqle_id = items[(items['Type'] == 'SQLEndpoint') & (items['Display Name']==lhname)]['Id'].values[0]\n","\n","\n","    # obtain the connection string for the warehouse to pass to the copy pipeline    \n","    whurl  = \"v1/workspaces/\" + target_ws_id + \"/warehouses/\" + target_wh_id\n","    whresponse = client.get(whurl)\n","    whconnStr = whresponse.json()['properties']['connectionInfo']\n","\n","    # obtain the pipeline id created to recover this warehouse\n","    plid = getItemId( target_ws_id,recovery_pipeline,'DataPipeline')\n","    if plid == 'NotExists':\n","        print('Error: Could not execute pipeline '+recovery_pipeline+ ' as the ID could not be obtained ')\n","    else:\n","        # pipeline url including pipeline Id unique to each warehouse\n","        plurl = 'v1/workspaces/'+target_ws_id+'/items/'+plid+'/jobs/instances?jobType=Pipeline'\n","        #print(plurl)\n","\n","        payload_data = '{' \\\n","            '\"executionData\": {' \\\n","                '\"parameters\": {' \\\n","                    '\"lakehouseId\": \"' + temp_lh_sqle_id + '\",' \\\n","                    '\"tablesToCopy\": ' + tablesToCopyParam + ',' \\\n","                    '\"workspaceId\": \"' + target_ws_id +'\",' \\\n","                    '\"warehouseId\": \"' + target_wh_id + '\",' \\\n","                    '\"lakehouseConnStr\": \"' + lhconnStr + '\",' \\\n","                    '\"warehouseConnStr\": \"' + whconnStr + '\"' \\\n","                    '}}}'\n","        #print(payload_data)\n","        plresponse = client.post(plurl, json=json.loads(payload_data))\n","        if p_logging_verbose:\n","            print(str(plresponse.status_code))      \n","print('Done')\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"collapsed":false},"id":"57dafef7-17a2-475f-9e62-eecc6660440c"},{"cell_type":"markdown","source":["###### Update direct lake model lakehouse connection\n","\n","https://semantic-link-labs.readthedocs.io/en/stable/sempy_labs.directlake.html#sempy_labs.directlake.update_direct_lake_model_lakehouse_connection\n","    "],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"cc97be77-116e-4cde-bdc6-2971ab98a083"},{"cell_type":"code","source":["\n","df_datasets = fabric.list_datasets(target_ws)\n","\n","# Iterate over each dataset in the dataframe\n","for index, row in df_datasets.iterrows():\n","    # Check if the dataset is not the default semantic model\n","    if not labs.is_default_semantic_model(row['Dataset Name'], fabric.resolve_workspace_id(target_ws)):\n","        print('Updating semantic model connection ' + row['Dataset Name'] + ' in workspace '+ target_ws)\n","        labs.directlake.update_direct_lake_model_connection(dataset=row['Dataset Name'], \n","                                                                        workspace= target_ws,\n","                                                                        source=labs.directlake.get_direct_lake_source(row['Dataset Name'], workspace= target_ws)[1], \n","                                                                        source_type=labs.directlake.get_direct_lake_source(row['Dataset Name'], workspace= target_ws)[0], \n","                                                                        source_workspace=target_ws)\n","        labs.refresh_semantic_model(dataset=row['Dataset Name'], workspace= target_ws)\n","\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"9deccda6-5c3d-4b88-8ed8-68855ca0949a"},{"cell_type":"markdown","source":["##### Rebind reports in new branch workspace\n","\n","https://semantic-link-labs.readthedocs.io/en/latest/sempy_labs.report.html#sempy_labs.report.report_rebind"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"36783f3b-4904-4d74-842d-dbd026a3184a"},{"cell_type":"code","source":["df_reports = fabric.list_reports(workspace=target_ws)\n","for index, row in df_reports.iterrows():\n","    #print(row['Name'] + '-' + row['Dataset Id'])\n","    df_datasets = fabric.list_datasets(workspace=target_ws)\n","    dataset_name = df_datasets[df_datasets['Dataset ID'] == row['Dataset Id']]['Dataset Name'].values[0]\n","    print(f'Rebinding report to {dataset_name} in {target_ws}')\n","    labs.report.report_rebind(report=row['Name'],dataset=dataset_name, report_workspace=target_ws, dataset_workspace=target_ws)\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"collapsed":false},"id":"06268ede-b795-493e-9a8d-772654ce7e20"},{"cell_type":"markdown","source":["##### Update data pipeline source & sink connections\n","\n","Support changes lakehouses, warehouses, notebooks and connections from source to target. <br>\n","Connections changes should be expressed as an array of tuples [{from_1:to_1},{from_N:to_N}]"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"4ae65012-350c-40c0-a68a-4069c567a85f"},{"cell_type":"code","source":["from typing import Optional\n","from sempy_labs._helper_functions import (\n","    resolve_workspace_name_and_id,\n","    lro,\n","    _decode_b64,\n",")\n","import sempy_labs._icons as icons\n","\n","import base64\n","from typing import Optional, Tuple, List\n","from uuid import UUID\n","\n","\n","def update_data_pipeline_definition(\n","    name: str, pipeline_content: dict, workspace: Optional[str] = None\n","):\n","    \"\"\"\n","    Updates an existing data pipeline with a new definition.\n","\n","    Parameters\n","    ----------\n","    name : str\n","        The name of the data pipeline.\n","    pipeline_content : dict\n","        The data pipeline content (not in Base64 format).\n","    workspace : str, default=None\n","        The name of the workspace.\n","        Defaults to None which resolves to the workspace of the attached lakehouse\n","        or if no lakehouse attached, resolves to the workspace of the notebook.\n","    \"\"\"\n","\n","    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)\n","    client = fabric.FabricRestClient()\n","    pipeline_payload = base64.b64encode(json.dumps(pipeline_content).encode('utf-8')).decode('utf-8')\n","    pipeline_id = fabric.resolve_item_id(\n","        item_name=name, type=\"DataPipeline\", workspace=workspace\n","    )\n","\n","    request_body = {\n","        \"definition\": {\n","            \"parts\": [\n","                {\n","                    \"path\": \"pipeline-content.json\",\n","                    \"payload\": pipeline_payload,\n","                    \"payloadType\": \"InlineBase64\"\n","                }\n","            ]\n","        }\n","    }\n","\n","\n","    response = client.post(\n","        f\"v1/workspaces/{workspace_id}/items/{pipeline_id}/updateDefinition\",\n","        json=request_body,\n","    )\n","\n","    lro(client, response, return_status_code=True)\n","\n","    print(\n","        f\"{icons.green_dot} The '{name}' pipeline was updated within the '{workspace}' workspace.\"\n","    )\n","\n","def _is_valid_uuid(\n","    guid: str,\n","):\n","    \"\"\"\n","    Validates if a string is a valid GUID in version 4\n","\n","    Parameters\n","    ----------\n","    guid : str\n","        GUID to be validated.\n","\n","    Returns\n","    -------\n","    bool\n","        Boolean that indicates if the string is a GUID or not.\n","    \"\"\"\n","\n","    try:\n","        UUID(str(guid), version=4)\n","        return True\n","    except ValueError:\n","        return False"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"bdd46d4a-ef58-4f9a-b2e8-a428361a17c1"},{"cell_type":"code","source":["import json\n","from jsonpath_ng import jsonpath, parse\n","from typing import Optional, Tuple, List\n","from uuid import UUID\n","\n","source_ws = ''\n","target_ws = ''\n","\n","\n","# Swaps the connection properties of an activity belonging to the specified item type(s)\n","def swap_pipeline_connection(pl_json: dict, p_source_ws: str,p_target_ws: str, \n","                                p_item_type: List =['DataWarehouse','Lakehouse','Notebook'], \n","                                p_conn_id_from_to: Optional[List[Tuple[str,str]]]=[]):\n","    \n","    source_ws_id = fabric.resolve_workspace_id(source_ws)\n","\n","    target_ws_id = fabric.resolve_workspace_id(target_ws)\n","\n","    if 'Warehouse' in p_item_type or 'Lakehouse' in p_item_type:\n","        ls_expr = parse('$..linkedService')\n","        for endpoint_match in ls_expr.find(pl_json):\n","            if endpoint_match.value['properties']['type'] == 'DataWarehouse' \\\n","                and endpoint_match.value['properties']['typeProperties']['workspaceId'] == source_ws_id \\\n","                and 'Warehouse' in p_item_type:\n","                # only update the warehouse if it was located in the source workspace i.e. we will update the properties to the target workspace if the warehouse resided in the same workspace as the pipeline\n","                #print(endpoint_match.value)\n","                warehouse_id = endpoint_match.value['properties']['typeProperties']['artifactId']\n","                #print(warehouse_id)\n","                warehouse_endpoint = endpoint_match.value['properties']['typeProperties']['endpoint']\n","                #print(warehouse_endpoint)\n","                \n","                source_wh_name = fabric.resolve_item_name(item_id = warehouse_id,workspace=source_ws_id)\n","                #print(remote_wh_name)\n","                # find the warehouse id of the warehouse with the same name in the target workspace\n","                target_wh_id = fabric.resolve_item_id(item_name = source_wh_name,type='Warehouse',workspace=target_ws_id)\n","                # look up the connection string for the warehouse in the target workspace\n","                whurl  = f\"v1/workspaces/{target_ws_id}/warehouses/{target_wh_id}\"\n","                whresponse = client.get(whurl)\n","                lhconnStr = whresponse.json()['properties']['connectionString']\n","                endpoint_match.value['properties']['typeProperties']['artifactId'] = target_wh_id\n","                endpoint_match.value['properties']['typeProperties']['workspaceId'] = target_ws_id\n","                endpoint_match.value['properties']['typeProperties']['endpoint'] = lhconnStr\n","                #print(endpoint_match.value)\n","                ls_expr.update(endpoint_match,endpoint_match.value)\n","            if endpoint_match.value['properties']['type'] == 'Lakehouse' \\\n","                and endpoint_match.value['properties']['typeProperties']['workspaceId'] == source_ws_id \\\n","                and 'Lakehouse' in p_item_type:\n","                #print(endpoint_match.value)\n","                lakehouse_id = endpoint_match.value['properties']['typeProperties']['artifactId']\n","                remote_lh_name = fabric.resolve_item_name(item_id = lakehouse_id,workspace=source_ws_id)\n","                # find the lakehouse id of the lakehouse with the same name in the target workspace\n","                target_lh_id = fabric.resolve_item_id(item_name = remote_lh_name,type='Lakehouse',workspace=target_ws_id)\n","                endpoint_match.value['properties']['typeProperties']['artifactId'] = target_lh_id\n","                endpoint_match.value['properties']['typeProperties']['workspaceId'] = target_ws_id\n","                ls_expr.update(endpoint_match,endpoint_match.value)\n","                #    print(endpoint_match.value)\n","\n","\n","    if 'Notebook' in p_item_type: \n","        ls_expr = parse('$..activities')\n","\n","        for endpoint_match in ls_expr.find(pl_json):\n","            for activity in endpoint_match.value:\n","                #print(activity['type'])\n","                if activity['type']=='TridentNotebook' and 'Notebook' in p_item_type: #only update if the notebook was in the same workspace as the pipeline\n","                    print('change from '+activity['typeProperties']['workspaceId'])\n","                    source_nb_id = activity['typeProperties']['notebookId']\n","                    source_nb_name = fabric.resolve_item_name(item_id = source_nb_id,workspace=source_ws_id)\n","                    target_nb_id = fabric.resolve_item_id(item_name = source_nb_name,type='Notebook',workspace=target_ws_id)\n","                    activity['typeProperties']['notebookId']=target_nb_id\n","                    activity['typeProperties']['workspaceId']=target_ws_id\n","                    print('to notebook '+ target_nb_id)\n","                    #ls_expr.update(endpoint_match,endpoint_match.value)\n","\n","    if p_conn_from_to:\n","        for ti_conn_from_to in p_conn_from_to:\n","            if not _is_valid_uuid(ti_conn_from_to[0]):\n","                print('Connection from is string '+ str(ti_conn_from_to[0]))\n","                dfC_filt = df_conns[df_conns[\"Connection Name\"] == ti_conn_from_to[0]]       \n","                connId_from = dfC_filt['Connection Id'].iloc[0]     \n","            else:\n","                connId_from = ti_conn_from_to[0]\n","\n","            if not _is_valid_uuid(ti_conn_from_to[1]):\n","                print('Connection from is string '+ str(ti_conn_from_to[1]))\n","                dfC_filt = df_conns[df_conns[\"Connection Name\"] == ti_conn_from_to[1]]       \n","                connId_to = dfC_filt['Connection Id'].iloc[0]     \n","            else:\n","                connId_to = ti_conn_from_to[1]\n","\n","            ls_expr = parse('$..externalReferences')\n","            for externalRef in ls_expr.find(pl_json):\n","                if externalRef.value['connection']==connId_from:\n","                    print('Changing connection from '+str(connId_from))\n","                    externalRef.value['connection']=connId_to\n","                    ls_expr.update(externalRef,externalRef.value)\n","                    print('to '+str(connId_to))\n","\n","    return pl_json\n","\n","\n","\n","# loading a dataframe of connections to perform an ID lookup if required \n","df_conns = labs.list_connections()\n","\n","df_pipeline = labs.list_data_pipelines(target_ws)\n","for index, row in df_pipeline.iterrows():\n","    #print(labs.get_data_pipeline_definition(row['Data Pipeline Name'],target_ws))\n","    if row['Data Pipeline Name']=='plRecover_WH6_Prod2_Warehouse2_fixed':\n","        pipeline_json = json.loads(labs.get_data_pipeline_definition(row['Data Pipeline Name'],source_ws))\n","\n","        p_new_json = swap_pipeline_connection(pipeline_json, source_ws,target_ws,\n","                ['DataWarehouse','Lakehouse','Notebook'],\n","                [p_connections_from_to]) \n","        #print(json.dumps(pipeline_json, indent=4))\n","        \n","        update_data_pipeline_definition(name=row['Data Pipeline Name'],pipeline_content=pipeline_json, workspace=target_ws)\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"079958e8-2880-484a-a994-41caf47e747e"},{"cell_type":"markdown","source":["##### Commit changes made above to Git"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"44174276-b983-4e80-9451-0afb9589cf1f"},{"cell_type":"code","source":["labs.commit_to_git(comment='Initial',  workspace=target_ws)"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"9a5c3d84-f71d-4348-b419-c4953ac9e1d0"}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"name":"synapse_pyspark","language":"Python","display_name":"Synapse PySpark"},"language_info":{"name":"python"},"microsoft":{"language":"python","language_group":"synapse_pyspark","ms_spell_check":{"ms_spell_check_language":"en"}},"widgets":{},"nteract":{"version":"nteract-front-end@1.0.0"},"synapse_widget":{"version":"0.1","state":{}},"spark_compute":{"compute_id":"/trident/default","session_options":{"conf":{"spark.synapse.nbs.session.timeout":"1200000"}}},"dependencies":{"lakehouse":{}}},"nbformat":4,"nbformat_minor":5}
\ No newline at end of file
+{"cells":[{"cell_type":"markdown","source":["##### Branch out to new workspace notebook - post activity\n","\n","After cloning a workspace, this notebook will reconfigure any references to the old workspace by rebinding them to the new workspace. \n","\n","For example a pipeline referencing a warehouse or a default lakehouse of a notebook.\n","\n","This notebook runs post activity tasks can be run after [branch out to new workspace functionality](https://blog.fabric.microsoft.com/en-us/blog/introducing-new-branching-capabilities-in-fabric-git-integration) or the [custom AzDO script](https://github.com/microsoft/fabric-toolbox/blob/main/accelerators/CICD/Branch-out-to-new-workspace/AzDO/scripts/BranchOut-Feature-Workspace-Automation.py).\n","\n","Summary of post activities in order:\n","<ul>\n","<li>Default lakehouses and warehouse are updated to local lakehouse/warehouses</li>\n","<li>Either creates shortcuts in local lakehouse back to tables in the source lakehouse, or copies the data from source lakehouse. Set via parameter below.</li>\n","<li>Copy warehouse data. Set via parameter below</li>\n","<li>Changes directlake semantic model connections for semantic models to \"local\" lakehouse/warehouse</li> \n","<li>Rebinds reports to \"local\" semantic models</li>\n","<li>Changes pipeline lakehouse/warehouse references to local item</li>\n","<li>Ability to swap connections in pipelines from old to new</li>\n","<li>Commit changes to git</li>\n","</ul>\n","\n","Requirements:\n","<ul>\n","<li>Requires Semantic Link Labs installed by pip install below or added to environment library.</li>\n","<li>Requires JmesPath library for data pipeline JSON manipulation i.e. connection swaps.</li>\n","</ul>\n","\n","Limitations of current script:\n","\n","<ul>\n","<li>Does not recreate item shares or external shortcuts</li>\n","<li>Does not re-apply lakehouse SQL Endpoint or Warehouse object/row/column level security</li>\n","<li>Does not recreate data access roles in Lakehouse</li>\n","<li>Untested with Lakehouses where with schema support enabled</li>\n","</ul>\n","\n","\n"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"a98b6d0a-7a36-4116-ab0d-aa70144eb737"},{"cell_type":"markdown","source":["##### Install semantic link labs to support advanced functionality\n","\n","https://semantic-link-labs.readthedocs.io/en/latest/index.html\n","https://github.com/microsoft/semantic-link-labs/blob/main/README.md\n","\n"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"3b887bd6-a9c9-430f-b58f-b58a93f5ce29"},{"cell_type":"code","source":["%pip -q install semantic-link-labs\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":true},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"1b03316d-c088-4a0e-a2f0-44d45d112121"},{"cell_type":"markdown","source":["##### Install Jmespath to make data pipeline changes such as updating linked notebooks, warehouses and lakehouses "],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"8a74ed11-dd64-43bb-a735-906a947c8666"},{"cell_type":"code","source":["%pip install jmespath"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"c68be6ba-7648-457f-af82-f1987d12d7f7"},{"cell_type":"markdown","source":["##### Set parameters\n","Before running this notebook ensure these parameters are set correctly. If necessary these can be passed in via a data factory pipeline"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"dee81614-b92b-4242-890a-b11f97b1a640"},{"cell_type":"code","source":["source_ws = ''\n","target_ws = ''\n","\n","# Either copy lakehouse data or create shortcuts, set at most one of these to True \n","copy_lakehouse_data = True\n","create_lakehouse_shortcuts = False\n","\n","# Option to copy warehouse data if required\n","copy_warehouse_data = True\n","\n","# If false then shortcuts will be created. If you wish to create shortcuts based on a pattern match please set the param below\n","# enter pattern match for creating shortcuts - see https://github.com/arasdk/fabric-code-samples/blob/main/shortcuts/fabric_shortcut_creator.py \n","PATTERN_MATCH = [\"*\"]\n","_inlineInstallationEnabled = True\n","\n","# Set connections to be replaced from previous name or ID to new name or ID.\n","p_connections_from_to = ()#('https://api.fabric.microsoft.com/v1/workspaces/ admin','4498340c-27cf-4c6e-a025-00e5de6b0726'),('4498340c-27cf-4c6e-a025-00e5de6b0726','https://api.fabric.microsoft.com/v1/workspaces/ admin'),('https://api.fabric.microsoft.com/v1/workspaces/ admin','4498340c-27cf-4c6e-a025-00e5de6b0726')"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"tags":["parameters"]},"id":"90efaa4f-846d-4924-900e-258837a3467d"},{"cell_type":"markdown","source":["##### Library imports and fabric rest client setup\n","\n","https://learn.microsoft.com/en-us/python/api/semantic-link-sempy/sempy.fabric.fabricrestclient"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"4fb01e1d-ec4e-4c69-b544-66f6d8c5a475"},{"cell_type":"code","source":["import pandas as pd\n","import datetime, time\n","import re,json, fnmatch,os\n","import requests, base64\n","import sempy\n","import sempy.fabric as fabric\n","from sempy.fabric.exceptions import FabricHTTPException, WorkspaceNotFoundException\n","from pyspark.sql import DataFrame\n","from pyspark.sql.functions import col,current_timestamp,lit\n","import sempy_labs as labs\n","from sempy_labs import migration, directlake\n","from sempy_labs import lakehouse as lake\n","from sempy_labs import report as rep\n","from sempy_labs.tom import connect_semantic_model\n","\n","# instantiate the Fabric rest client\n","client = fabric.FabricRestClient()\n","\n","# get the current workspace ID based on the context of where this notebook is run from\n","thisWsId = notebookutils.runtime.context['currentWorkspaceId']\n","thisWsName = notebookutils.runtime.context['currentWorkspaceName']\n","\n","source_ws_id = fabric.resolve_workspace_id(source_ws)\n","target_ws_id = fabric.resolve_workspace_id(target_ws)\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"391624c1-b299-452d-9ebf-f32626d49970"},{"cell_type":"markdown","source":["##### Update default and attached lakehouses/warehouses for notebooks\n","\n","Update notebook dependencies based on but now supports T-SQL notebooks:\n","https://github.com/PowerBiDevCamp/FabConWorkshopSweden/blob/main/DemoFiles/GitUpdateWorkspace/updateWorkspaceDependencies_v1.ipynb\n"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"aaae8a08-588d-4dd8-9d2c-2200b7a88d30"},{"cell_type":"code","source":["for notebook in notebookutils.notebook.list(workspaceId=target_ws_id):\n","    updates = False\n","    if notebook.displayName == 'ETL':#True: #notebook.displayName == 'T-SQL_Notebook': #notebook.displayName != 'Create Feature Branch':\n","\n","        # Get the current notebook definition\n","        json_payload = json.loads(notebookutils.notebook.getDefinition(notebook.displayName,workspaceId=source_ws_id))\n","        #print(json.dumps(json_payload, indent=4))\n","        # Check for any attached lakehouses\n","        if 'dependencies' in json_payload['metadata'] \\\n","            and 'lakehouse' in json_payload['metadata']['dependencies'] \\\n","            and json_payload['metadata'][\"dependencies\"][\"lakehouse\"] is not None:\n","            # Extract attached and default lakehouses\n","            current_lakehouse = json_payload['metadata']['dependencies']['lakehouse']\n","            # if default lakehouse setting exists\n","            if 'default_lakehouse_name' in current_lakehouse:\n","                print(f\"Updating notebook {notebook.displayName} with new default lakehouse: {current_lakehouse['default_lakehouse_name']} in workspace {target_ws}\")\n","                source_lh_name = fabric.resolve_item_name(item_id = current_lakehouse['default_lakehouse'],type='Lakehouse',workspace=source_ws_id)\n","                current_lakehouse['default_lakehouse'] = fabric.resolve_item_id(item_name = source_lh_name,type='Lakehouse',workspace=target_ws_id)\n","                current_lakehouse['default_lakehouse_workspace_id'] = target_ws_id\n","                updates = True\n","            # loop through all attached lakehouess\n","            for lakehouse in json_payload['metadata']['dependencies']['lakehouse']['known_lakehouses']:\n","                source_lh_id = lakehouse['id']\n","                # find source lakehouse name\n","                source_lh_name = fabric.resolve_item_name(item_id = lakehouse['id'],type='Lakehouse',workspace=source_ws_id)\n","                # find target lakehouse id based on name\n","                target_lh_id = fabric.resolve_item_id(item_name = source_lh_name,type='Lakehouse',workspace=target_ws_id)\n","                lakehouse['id'] = target_lh_id\n","                print(f'Updating attached lakehouse {source_lh_name} from {source_lh_id} to target ID {target_lh_id}')\n","                updates = True\n","\n","        if 'dependencies' in json_payload['metadata'] and 'warehouse' in json_payload['metadata']['dependencies']:\n","            # Fetch existing details\n","            current_warehouse = json_payload['metadata']['dependencies']['warehouse']\n","            current_warehouse_id = current_warehouse['default_warehouse']\n","            source_wh_name =  fabric.resolve_item_name(item_id = current_warehouse_id,workspace=source_ws_id)\n","            #print('Source warehouse name is ' + source_wh_name)\n","            target_wh_id = fabric.resolve_item_id(item_name = source_wh_name,type='Warehouse',workspace=target_ws_id)\n","\n","            if 'default_warehouse' in current_warehouse:\n","                #json_payload['metadata']['dependencies']['warehouse'] = {}\n","                print(f\"Attempting to update notebook {notebook.displayName} with new default warehouse: {target_wh_id} in {target_ws}\")\n","            \n","                json_payload['metadata']['dependencies']['warehouse']['default_warehouse'] = target_wh_id\n","                for warehouse in json_payload['metadata']['dependencies']['warehouse']['known_warehouses']:\n","                    if warehouse['id'] == current_warehouse_id:\n","                        warehouse['id'] = target_wh_id\n","                        updates = True\n","\n","        if updates:\n","            notebookutils.notebook.updateDefinition(\n","                    name = notebook.displayName,\n","                    content  = json.dumps(json_payload),\n","                    workspaceId = target_ws_id\n","                    )\n","            \n","            print(f\"Updated notebook {notebook.displayName} in {target_ws}\")\n","\n","        else:\n","            print(f'No default lakehouse set for notebook {notebook.displayName}, ignoring.')"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"5c60b5d2-f83c-46f8-9870-9fd609166b67"},{"cell_type":"markdown","source":["##### Run the below cell - contains utility functions to support lakehouse and warehouse initialisation\n","\n","Shortcut creator:\n","https://github.com/arasdk/fabric-code-samples/blob/main/shortcuts/fabric_shortcut_creator.py "],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"a47a56df-219d-4d6c-b950-491909638deb"},{"cell_type":"code","source":["##### \n","### Shortcut utility function \n","####\n","\n","# Extract workspace_id, item_id and path from a onelake URI\n","def extract_onelake_https_uri_components(uri):\n","    # Define a regular expression to match any string between slashes and capture the final path element(s) without the leading slash\n","    pattern = re.compile(r\"abfss://([^@]+)@[^/]+/([^/]+)/(.*)\")\n","    match = pattern.search(uri)\n","    if match:\n","        workspace_id, item_id, path = match.groups()\n","        return workspace_id, item_id, path\n","    else:\n","        return None, None, None\n","\n","\n","def is_valid_onelake_uri(uri: str) -> bool:\n","    workspace_id, item_id, path = extract_onelake_https_uri_components(uri)\n","    if \"abfss://\" not in uri or workspace_id is None or item_id is None or path is None:\n","        return False\n","\n","    return True\n","\n","\n","def get_last_path_segment(uri: str):\n","    path = uri.split(\"/\")  # Split the entire URI by '/'\n","    return path[-1] if path else None\n","\n","\n","def is_delta_table(uri: str):\n","    delta_log_path = os.path.join(uri, \"_delta_log\")\n","    return mssparkutils.fs.exists(delta_log_path)\n","\n","\n","def get_onelake_shorcut(workspace_id: str, item_id: str, path: str, name: str):\n","    shortcut_uri = (\n","        f\"v1/workspaces/{workspace_id}/items/{item_id}/shortcuts/{path}/{name}\"\n","    )\n","    result = client.get(shortcut_uri).json()\n","    return result\n","\n","\n","def is_folder_matching_pattern(path: str, folder_name: str, patterns: []):\n","    if folder_name in patterns:\n","        return True\n","    else:\n","        for pattern in patterns:\n","            if fnmatch.fnmatch(folder_name, pattern):\n","                return is_delta_table(path)\n","\n","    return False\n","\n","\n","def get_matching_delta_tables_uris(uri: str, patterns: []) -> []:\n","    # Use a set to avoid duplicates\n","    matched_uris = set()\n","    files = mssparkutils.fs.ls(uri)\n","    folders = [item for item in files if item.isDir]\n","\n","    # Filter folders to only those that matches the pattern and is a delta table\n","    matched_uris.update(\n","        folder.path\n","        for folder in folders\n","        if is_folder_matching_pattern(folder.path, folder.name, patterns)\n","    )\n","\n","    return matched_uris\n","\n","\n","def create_onelake_shorcut(source_uri: str, dest_uri: str):\n","    src_workspace_id, src_item_id, src_path = extract_onelake_https_uri_components(\n","        source_uri\n","    )\n","\n","    dest_workspace_id, dest_item_id, dest_path = extract_onelake_https_uri_components(\n","        dest_uri\n","    )\n","\n","    name = get_last_path_segment(source_uri)\n","    dest_uri_joined = os.path.join(dest_uri, name)\n","\n","    # If the destination path already exists, return without creating shortcut\n","    if mssparkutils.fs.exists(dest_uri_joined):\n","        print(f\"Destination already exists: {dest_uri_joined}\")\n","        return None\n","\n","    request_body = {\n","        \"name\": name,\n","        \"path\": dest_path,\n","        \"target\": {\n","            \"oneLake\": {\n","                \"itemId\": src_item_id,\n","                \"path\": src_path,\n","                \"workspaceId\": src_workspace_id,\n","            }\n","        },\n","    }\n","\n","    shortcut_uri = f\"v1/workspaces/{dest_workspace_id}/items/{dest_item_id}/shortcuts\"\n","    print(f\"Creating shortcut: {shortcut_uri}/{name}..\")\n","    try:\n","        client.post(shortcut_uri, json=request_body)\n","    except FabricHTTPException as e:\n","        print(e)\n","        return None\n","\n","    return get_onelake_shorcut(dest_workspace_id, dest_item_id, dest_path, name)\n","   \n","\n","####\n","## Copy lakehouse and warehouse utility functions\n","####\n","\n","def get_lh_object_list(base_path,data_types = ['Tables', 'Files'])->pd.DataFrame:\n","\n","    '''\n","    Function to get a list of tables for a lakehouse\n","    adapted from https://fabric.guru/getting-a-list-of-folders-and-delta-tables-in-the-fabric-lakehouse\n","    This function will return a pandas dataframe containing names and abfss paths of each folder for Files and Tables\n","    '''\n","    #data_types = ['Tables', 'Files'] #for if you want a list of files and tables\n","    #data_types = ['Tables'] #for if you want a list of tables\n","\n","    df = pd.concat([\n","        pd.DataFrame({\n","            'name': [item.name for item in notebookutils.fs.ls(f'{base_path}/{data_type}/')],\n","            'type': data_type[:-1].lower() , \n","            'src_path': [item.path for item in notebookutils.fs.ls(f'{base_path}/{data_type}/')],\n","        }) for data_type in data_types], ignore_index=True)\n","\n","    return df\n","\n","def get_wh_object_list(schema_list,base_path)->pd.DataFrame:\n","\n","    '''\n","    Function to get a list of tables for a warehouse by schema\n","    '''\n","    data_type = 'Tables'\n","    dfs = []\n","\n","    for schema_prefix in schema_list:\n","        if notebookutils.fs.exists(f'{base_path}/{data_type}/{schema_prefix}/'):\n","            items = notebookutils.fs.ls(f'{base_path}/{data_type}/{schema_prefix}/')\n","            if items:  # Check if the list is not empty\n","                df = pd.DataFrame({\n","                    'schema': schema_prefix,\n","                    'name': [item.name for item in items],\n","                    'type': data_type[:-1].lower(),\n","                    'src_path': [item.path for item in items],\n","                })\n","                dfs.append(df)\n","\n","    if dfs:  # Check if the list of dataframes is not empty\n","        df = pd.concat(dfs, ignore_index=True)\n","    else:\n","        df = pd.DataFrame()  # Return an empty dataframe if no dataframes were created\n","\n","    return df\n","\n","def copy_lh_objects(table_list,workspace_src,workspace_tgt,lakehouse_src,lakehouse_tgt,fastcopy=True,usingIDs=False)->pd.DataFrame:\n","    # declare an array to keep the instrumentation\n","    cpresult = []\n","    # loop through all the tables to extract the source path \n","    for table in table_list.src_path:\n","        source = table\n","        destination = source.replace(f'abfss://{workspace_src}', f'abfss://{workspace_tgt}')\n","        if usingIDs:\n","            destination = destination.replace(f'{lakehouse_src}', f'{lakehouse_tgt}')\n","        else:\n","            destination = destination.replace(f'{lakehouse_src}.Lakehouse', f'{lakehouse_tgt}.Lakehouse')\n","        start_time =  datetime.datetime.now()\n","        if notebookutils.fs.exists(destination):\n","             notebookutils.fs.rm(destination, True)\n","        if fastcopy:\n","            # use fastcopy util which is a python wrapper to azcopy\n","            notebookutils.fs.fastcp(source+'/*', destination+'/', True)\n","        else:\n","            notebookutils.fs.cp(source, destination, True)\n","\n","        # recording the timing and add it to the results list\n","        end_time = datetime.datetime.now()\n","        copyreslist = [source, destination, start_time.strftime(\"%Y-%m-%d %H:%M:%S\"),  end_time.strftime(\"%Y-%m-%d %H:%M:%S\"), str((end_time - start_time).total_seconds())]\n","        cpresult.append(copyreslist)\n","    return pd.DataFrame(cpresult,columns =['source--------------------------------------','target--------------------------------------','start------------','end_time------------','elapsed seconds----'])\n","\n","def createDWrecoverypl(ws_id,pl_name = 'Recover_Warehouse_Data_From_DR'):\n","  client = fabric.FabricRestClient()\n","\n","  dfurl= \"v1/workspaces/\"+ ws_id + \"/items\"\n","  payload = { \n","  \"displayName\": pl_name, \n","  \"type\": \"DataPipeline\", \n","  \"definition\": { \n","    \"parts\": [ \n","      { \n","        \"path\": \"pipeline-content.json\", \n","        \"payload\":  \"ewogICAgInByb3BlcnRpZXMiOiB7CiAgICAgICAgImFjdGl2aXRpZXMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICJuYW1lIjogIkl0ZXJhdGVTY2hlbWFUYWJsZXMiLAogICAgICAgICAgICAgICAgInR5cGUiOiAiRm9yRWFjaCIsCiAgICAgICAgICAgICAgICAiZGVwZW5kc09uIjogW10sCiAgICAgICAgICAgICAgICAidHlwZVByb3BlcnRpZXMiOiB7CiAgICAgICAgICAgICAgICAgICAgIml0ZW1zIjogewogICAgICAgICAgICAgICAgICAgICAgICAidmFsdWUiOiAiQHBpcGVsaW5lKCkucGFyYW1ldGVycy50YWJsZXNUb0NvcHkiLAogICAgICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJFeHByZXNzaW9uIgogICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgImJhdGNoQ291bnQiOiAyMCwKICAgICAgICAgICAgICAgICAgICAiYWN0aXZpdGllcyI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAgICAgIm5hbWUiOiAiQ29weVdhcmVob3VzZVRhYmxlcyIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJDb3B5IiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkZXBlbmRzT24iOiBbCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYWN0aXZpdHkiOiAiU2V0IHRhYmxlIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImRlcGVuZGVuY3lDb25kaXRpb25zIjogWwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIlN1Y2NlZWRlZCIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgXQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAicG9saWN5IjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0aW1lb3V0IjogIjAuMTI6MDA6MDAiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJyZXRyeSI6IDIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInJldHJ5SW50ZXJ2YWxJblNlY29uZHMiOiAzMCwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAic2VjdXJlT3V0cHV0IjogZmFsc2UsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInNlY3VyZUlucHV0IjogZmFsc2UKICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZVByb3BlcnRpZXMiOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInNvdXJjZSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInR5cGUiOiAiRGF0YVdhcmVob3VzZVNvdXJjZSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJxdWVyeVRpbWVvdXQiOiAiMDI6MDA6MDAiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAicGFydGl0aW9uT3B0aW9uIjogIk5vbmUiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZGF0YXNldFNldHRpbmdzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImFubm90YXRpb25zIjogW10sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibGlua2VkU2VydmljZSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibmFtZSI6ICIwN2EwMzAwNl9kMWI2XzRhMzlfYmViMV8wYmJhMmFhZjVmZjciLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJwcm9wZXJ0aWVzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYW5ub3RhdGlvbnMiOiBbXSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInR5cGUiOiAiRGF0YVdhcmVob3VzZSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0eXBlUHJvcGVydGllcyI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlbmRwb2ludCI6ICJAcGlwZWxpbmUoKS5wYXJhbWV0ZXJzLmxha2Vob3VzZUNvbm5TdHIiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImFydGlmYWN0SWQiOiAiQHBpcGVsaW5lKCkucGFyYW1ldGVycy5sYWtlaG91c2VJZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAid29ya3NwYWNlSWQiOiAiQHBpcGVsaW5lKCkucGFyYW1ldGVycy53b3Jrc3BhY2VJZCIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJEYXRhV2FyZWhvdXNlVGFibGUiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInNjaGVtYSI6IFtdLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInR5cGVQcm9wZXJ0aWVzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzY2hlbWEiOiAiZGJvIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidGFibGUiOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ2YWx1ZSI6ICJAY29uY2F0KGNvbmNhdChpdGVtKCkuc2NoZW1hLCdfJyksaXRlbSgpLm5hbWUpIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInR5cGUiOiAiRXhwcmVzc2lvbiIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzaW5rIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJEYXRhV2FyZWhvdXNlU2luayIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJhbGxvd0NvcHlDb21tYW5kIjogdHJ1ZSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInRhYmxlT3B0aW9uIjogImF1dG9DcmVhdGUiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiZGF0YXNldFNldHRpbmdzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImFubm90YXRpb25zIjogW10sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibGlua2VkU2VydmljZSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAibmFtZSI6ICIwYzAzMTIzYV9kMzEyXzQ2YzRfYThlN181YjRjYWQ4ZjEyZDciLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJwcm9wZXJ0aWVzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYW5ub3RhdGlvbnMiOiBbXSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInR5cGUiOiAiRGF0YVdhcmVob3VzZSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0eXBlUHJvcGVydGllcyI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJlbmRwb2ludCI6ICJAcGlwZWxpbmUoKS5wYXJhbWV0ZXJzLndhcmVob3VzZUNvbm5TdHIiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImFydGlmYWN0SWQiOiAiQHBpcGVsaW5lKCkucGFyYW1ldGVycy53YXJlaG91c2VJZCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAid29ya3NwYWNlSWQiOiAiQHBpcGVsaW5lKCkucGFyYW1ldGVycy53b3Jrc3BhY2VJZCIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJEYXRhV2FyZWhvdXNlVGFibGUiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInNjaGVtYSI6IFtdLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInR5cGVQcm9wZXJ0aWVzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzY2hlbWEiOiAiZGJvIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidGFibGUiOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ2YWx1ZSI6ICJAaXRlbSgpLm5hbWUiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJFeHByZXNzaW9uIgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImVuYWJsZVN0YWdpbmciOiB0cnVlLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0cmFuc2xhdG9yIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJUYWJ1bGFyVHJhbnNsYXRvciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0eXBlQ29udmVyc2lvbiI6IHRydWUsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0eXBlQ29udmVyc2lvblNldHRpbmdzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImFsbG93RGF0YVRydW5jYXRpb24iOiB0cnVlLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInRyZWF0Qm9vbGVhbkFzTnVtYmVyIjogZmFsc2UKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAgICAgIm5hbWUiOiAiU2V0IHRhYmxlIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0eXBlIjogIlNldFZhcmlhYmxlIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkZXBlbmRzT24iOiBbCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiYWN0aXZpdHkiOiAiU2V0IHNjaGVtYSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkZXBlbmRlbmN5Q29uZGl0aW9ucyI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJTdWNjZWVkZWQiCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICAgICAgICAgICAgICBdLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgInBvbGljeSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAic2VjdXJlT3V0cHV0IjogZmFsc2UsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInNlY3VyZUlucHV0IjogZmFsc2UKICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZVByb3BlcnRpZXMiOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInZhcmlhYmxlTmFtZSI6ICJUYWJsZW5hbWUiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ2YWx1ZSI6IHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInZhbHVlIjogIkBpdGVtKCkubmFtZSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0eXBlIjogIkV4cHJlc3Npb24iCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAibmFtZSI6ICJTZXQgc2NoZW1hIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ0eXBlIjogIlNldFZhcmlhYmxlIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICJkZXBlbmRzT24iOiBbXSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICJwb2xpY3kiOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInNlY3VyZU91dHB1dCI6IGZhbHNlLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzZWN1cmVJbnB1dCI6IGZhbHNlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgInR5cGVQcm9wZXJ0aWVzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJ2YXJpYWJsZU5hbWUiOiAiU2NoZW1hbmFtZSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInZhbHVlIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidmFsdWUiOiAiQGl0ZW0oKS5zY2hlbWEiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidHlwZSI6ICJFeHByZXNzaW9uIgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgfQogICAgICAgIF0sCiAgICAgICAgInBhcmFtZXRlcnMiOiB7CiAgICAgICAgICAgICJsYWtlaG91c2VJZCI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogInN0cmluZyIsCiAgICAgICAgICAgICAgICAiZGVmYXVsdFZhbHVlIjogIjBmMGY2YjdjLTE3NjEtNDFlNi04OTZlLTMwMDE0ZjE2ZmY2ZCIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgInRhYmxlc1RvQ29weSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogImFycmF5IiwKICAgICAgICAgICAgICAgICJkZWZhdWx0VmFsdWUiOiBbCiAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAic2NoZW1hIjogImRibyIsCiAgICAgICAgICAgICAgICAgICAgICAgICJuYW1lIjogIkRhdGUiCiAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgICAgICJzY2hlbWEiOiAiZGJvIiwKICAgICAgICAgICAgICAgICAgICAgICAgIm5hbWUiOiAiR2VvZ3JhcGh5IgogICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAic2NoZW1hIjogImRibyIsCiAgICAgICAgICAgICAgICAgICAgICAgICJuYW1lIjogIkhhY2tuZXlMaWNlbnNlIgogICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAic2NoZW1hIjogImRibyIsCiAgICAgICAgICAgICAgICAgICAgICAgICJuYW1lIjogIk1lZGFsbGlvbiIKICAgICAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAgICAgInNjaGVtYSI6ICJkYm8iLAogICAgICAgICAgICAgICAgICAgICAgICAibmFtZSI6ICJUaW1lIgogICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAic2NoZW1hIjogImRibyIsCiAgICAgICAgICAgICAgICAgICAgICAgICJuYW1lIjogIlRyaXAiCiAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgICAgICJzY2hlbWEiOiAiZGJvIiwKICAgICAgICAgICAgICAgICAgICAgICAgIm5hbWUiOiAiV2VhdGhlciIKICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICBdCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgICJ3b3Jrc3BhY2VJZCI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogInN0cmluZyIsCiAgICAgICAgICAgICAgICAiZGVmYXVsdFZhbHVlIjogIjE1MDExNDNjLTI3MmYtNGEyZi05NzZhLTdlNTU5NzFlNGMyYiIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgIndhcmVob3VzZUlkIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAic3RyaW5nIiwKICAgICAgICAgICAgICAgICJkZWZhdWx0VmFsdWUiOiAiNGQxYmQ5NTEtOTlkZS00YmQ3LWI3YmMtNzFjOGY1NmRiNDExIgogICAgICAgICAgICB9LAogICAgICAgICAgICAid2FyZWhvdXNlQ29ublN0ciI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogInN0cmluZyIsCiAgICAgICAgICAgICAgICAiZGVmYXVsdFZhbHVlIjogIjcyd3diaXZpMnViZWpicnRtdGFobzMyYjR5LWhxa2FjZmpwZTR4dXZmM2twemt6b2hzbWZtLmRhdGF3YXJlaG91c2UuZmFicmljLm1pY3Jvc29mdC5jb20iCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgICJsYWtlaG91c2VDb25uU3RyIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAic3RyaW5nIiwKICAgICAgICAgICAgICAgICJkZWZhdWx0VmFsdWUiOiAiNzJ3d2JpdmkydWJlamJydG10YWhvMzJiNHktaHFrYWNmanBlNHh1dmYza3B6a3pvaHNtZm0uZGF0YXdhcmVob3VzZS5mYWJyaWMubWljcm9zb2Z0LmNvbSIKICAgICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgInZhcmlhYmxlcyI6IHsKICAgICAgICAgICAgIlRhYmxlbmFtZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIlN0cmluZyIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgIlNjaGVtYW5hbWUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJTdHJpbmciCiAgICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgICJsYXN0TW9kaWZpZWRCeU9iamVjdElkIjogIjRhYTIwYWY3LTk0YmQtNDM0OC1iZWY4LWY4Y2JjZDg0MGQ1MSIsCiAgICAgICAgImxhc3RQdWJsaXNoVGltZSI6ICIyMDI0LTExLTEzVDE1OjUyOjUyWiIKICAgIH0KfQ==\", \n","        \"payloadType\": \"InlineBase64\" \n","      } \n","    ] \n","  } \n","}   \n","  \n","  response = json.loads(client.post(dfurl,json= payload).content)\n","  return response['id']\n","\n","def getItemId(wks_id,itm_name,itm_type):\n","    df = fabric.list_items(type=None,workspace=wks_id)\n","    #print(df)\n","    if df.empty:\n","        return 'NotExists'\n","    else:\n","        #display(df)\n","        #print(df.query('\"Display Name\"=\"'+itm_name+'\"'))\n","        if itm_type != '':\n","            newdf= df.loc[(df['Display Name'] == itm_name) & (df['Type'] == itm_type)]['Id']\n","        else:\n","            newdf= df.loc[(df['Display Name'] == itm_name)]['Id']  \n","        if newdf.empty:\n","            return 'NotExists'\n","        else:\n","            return newdf.iloc[0]\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":true,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"collapsed":false},"id":"e46210e9-58c9-483a-84ae-bbdc2ad1c37f"},{"cell_type":"markdown","source":["##### Either create shortcuts from source to target lakehouse(s) or copy data\n","\n","Loops through lakehouse(s) in the target workspace and either populates them with shortcuts or data\n","\n"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"a15065f3-670d-4bc9-b337-51709f6cdb1f"},{"cell_type":"code","source":["df_lhs = labs.list_lakehouses(source_ws)\n","for index, row in df_lhs.iterrows():\n","\n","\n","    if copy_lakehouse_data:\n","        df_lakehouses = (labs.list_lakehouses(source_ws))\n","        lh_name= row['Lakehouse Name']\n","        if lh_name.find('temp')==-1:\n","            # Gathers the list of recovers tables and source paths to be copied into the lakehouse associated with this notebook \n","            src_path = f'abfss://{source_ws}@onelake.dfs.fabric.microsoft.com/{lh_name}.Lakehouse'\n","\n","            table_list = get_lh_object_list(src_path)\n","            print(f'Attempting to copy table data for lakehouse {lh_name} from workspace {source_ws} to {target_ws}...')\n","            display(table_list)\n","\n","            #print('Copy Lakehouse Delta tables...')\n","            res = copy_lh_objects(table_list[table_list['type']=='table'],source_ws,target_ws,\n","                                lh_name,lh_name,False,False)\n","            display(res)\n","            # Copy files\n","            print(f'Attempting to copy file data for lakehouse {lh_name} from workspace {source_ws} to {target_ws}...')\n","\n","            #print('Copy Lakehouse files...')\n","            res = copy_lh_objects(table_list[table_list['type']=='file'],source_ws,target_ws,\n","                                lh_name,lh_name,False,False)\n","            display(res)\n","            print('Done.')\n","\n","    else:\n","        # fetch ID of source lakehouse based on name and workspace\n","        source_lh_id = fabric.resolve_item_id(\n","            item_name=row['Lakehouse Name'], type=\"Lakehouse\", workspace=source_ws\n","            )\n","        #target_lh_id = notebookutils.lakehouse.getWithProperties(name=current_lakehouse['default_lakehouse_name'], workspaceId=new_workspace_id)['id']\n","\n","        SOURCE_URI = f\"abfss://{source_ws_id}@onelake.dfs.fabric.microsoft.com/{source_lh_id}/Tables\"\n","        DEST_URI = f\"abfss://{target_ws_id}@onelake.dfs.fabric.microsoft.com/{row['Lakehouse ID']}/Tables\"\n","\n","        if PATTERN_MATCH is None or len(PATTERN_MATCH) == 0:\n","            raise TypeError(\"Argument 'PATTERN_MATCH' should be a valid list of patterns or [\"*\"] to match everything\")\n","\n","        # Collect created shortcuts\n","        result = []\n","\n","        # If either URI's are invalid, just return\n","        if not is_valid_onelake_uri(SOURCE_URI) or not is_valid_onelake_uri(DEST_URI):\n","            print(\n","                \"invalid URI's provided. URI's should be in the form: abfss://<workspace-id>@onelake.dfs.fabric.microsoft.com/<item-id>/<path>\"\n","            )\n","        else:\n","            # Remove any trailing '/' from uri's\n","            source_uri_addr = SOURCE_URI.rstrip(\"/\")\n","            dest_uri_addr = DEST_URI.rstrip(\"/\")\n","\n","            dest_workspace_id, dest_item_id, dest_path = extract_onelake_https_uri_components(\n","                dest_uri_addr\n","            )\n","\n","            # If we are not shortcutting to a managed table folder or\n","            # the source uri is a delta table, just shortcut it 1-1.\n","            if not dest_path.startswith(\"Tables\") or is_delta_table(source_uri_addr):\n","                shortcut = create_onelake_shorcut(source_uri_addr, dest_uri_addr)\n","                if shortcut is not None:\n","                    result.append(shortcut)\n","            else:\n","                # If source is not a delta table, and destination is managed table folder:\n","                # Iterate over source folders and create table shortcuts @ destination\n","                for delta_table_uri in get_matching_delta_tables_uris(\n","                    source_uri_addr, PATTERN_MATCH\n","                ):\n","                    shortcut = create_onelake_shorcut(delta_table_uri, dest_uri_addr)\n","                    if shortcut is not None:\n","                        result.append(shortcut)\n","        print(result)\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"collapsed":false},"id":"8bec3fbc-4a75-4ba3-86d5-0620ec504a8f"},{"cell_type":"markdown","source":["##### Copy warehouse data via parameterised pipeline\n","\n","Loop through all warehouses and copy the data"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"f198b816-77e9-4f04-9139-d78237bedc72"},{"cell_type":"code","source":["p_logging_verbose = True\n","df_warehouses = (labs.list_warehouses(target_ws))\n","display(df_warehouses)\n","for index, row in df_warehouses.iterrows():\n","    source_wh_id = labs.resolve_warehouse_id(row['Warehouse Name'],source_ws_id)\n","    target_wh_id = labs.resolve_warehouse_id(row['Warehouse Name'],target_ws_id)\n","    \n","    src_path = f'abfss://'+source_ws_id+'@onelake.dfs.fabric.microsoft.com/'+source_wh_id\n","    tgt_path = f'abfss://'+target_ws_id+'@onelake.dfs.fabric.microsoft.com/'+target_wh_id\n","\n","    # extract the list of schemas per data \n","    schema_list = get_lh_object_list(src_path,['Tables'])\n","    # extract a list of warehouse objects per schema and store in a list\n","    table_list = get_wh_object_list(schema_list['name'],src_path)\n","  \n","    # create a temporary staging lakehouse per warehouse to create shortcuts into, \n","    # which point back to original warehouse data currently in the DR storage account\n","    lhname = 'temp_rlh_' + source_ws+'_'+row['Warehouse Name']\n","    # check if it exists before attempting create\n","    if p_logging_verbose:\n","        print('Checking whether the temporary lakehouse \"'+ lhname +'\" exists in workspace '+target_ws+'...')\n","    temp_lh_id = getItemId(target_ws_id,lhname,'Lakehouse')\n","    if temp_lh_id == 'NotExists':\n","        lhname = lhname[:256] # lakehouse name should not exceed 256 characters\n","        payload = payload = '{\"displayName\": \"' + lhname + '\",' \\\n","        + '\"description\":  \"Interim staging lakehouse for primary warehouse recovery: ' \\\n","        + source_ws+'_'+row['Warehouse Name'] + 'into workspace '+ target_ws + '(' + target_ws +')\"}'\n","        try:\n","            lhurl = \"v1/workspaces/\" + target_ws_id + \"/lakehouses\"\n","            lhresponse = client.post(lhurl,json= json.loads(payload))\n","            temp_lh_id = lhresponse.json()['id']\n","            if p_logging_verbose:\n","                print('Temporary lakehouse \"'+ lhname +'\" created with Id ' + temp_lh_id + ': ' + str(lhresponse.status_code) + ' ' + str(lhresponse.text))\n","        except Exception as error:\n","            print(error.errorCode)\n","    else:\n","        if p_logging_verbose:\n","            print('Temporary lakehouse '+lhname+' (' + temp_lh_id + ') already exists.')\n","        \n","    time.sleep(60) # waiting for temporary lakehouse to provision completely   \n","\n","    # Create shortcuts for every table in the format of schema_table under the tables folder\n","    for index,itable in table_list.iterrows():\n","        shortcutExists=False\n","        # Check if shortcut exists\n","        try:\n","            url = \"v1/workspaces/\" + target_ws_id + \"/items/\" + temp_lh_id + \"/shortcuts/Tables/\"+itable['schema']+'_'+itable['name']\n","            tlhresponse = client.get(url)\n","            shortcutExists = True\n","            if p_logging_verbose:\n","                print('Shortcut '+itable['schema']+'_'+itable['name'] +' already exists')\n","        except Exception as error:\n","            shortcutExists = False    \n","\n","        if not shortcutExists: \n","            # Create shortcuts - one per table per schema\n","            url = \"v1/workspaces/\" + target_ws_id + \"/items/\" + temp_lh_id + \"/shortcuts\"\n","            scpayload = '{' \\\n","            '\"path\": \"Tables/\",' \\\n","            '\"name\": \"'+itable['schema']+'_'+itable['name']+'\",' \\\n","            '\"target\": {' \\\n","            '\"oneLake\": {' \\\n","                '\"workspaceId\": \"' + source_ws_id + '\",' \\\n","                '\"itemId\": \"'+ source_wh_id +'\",' \\\n","                '\"path\": \"/Tables/' + itable['schema']+'/'+itable['name'] + '\"' \\\n","                '}}}' \n","            try:\n","                #print(scpayload)                \n","                shctresponse = client.post(url,json= json.loads(scpayload))\n","                if p_logging_verbose:\n","                    print('Shortcut '+itable['schema']+'_'+itable['name'] + ' created.' )\n","\n","            except Exception as error:\n","                print('Error creating shortcut '+itable['schema']+'_'+itable['name']+' due to '+str(error) + ':' + shctresponse.text)\n","    \n","    recovery_pipeline_prefix= 'plRecover_WH'       \n","    # recovery pipeline name should not exceed 256 characters\n","    recovery_pipeline = recovery_pipeline_prefix+'_'+source_ws + '_'+row['Warehouse Name'][:256]\n","    if p_logging_verbose:\n","        print('Attempting to deploy a copy pipeline in the target workspace to load the target warehouse tables from the shortcuts created above... ')\n","    # Create the pipeline in the target workspace that loads the target warehouse from shortcuts created above \n","    plid = getItemId( target_ws_id,recovery_pipeline,'DataPipeline')\n","    #print(plid)\n","    if plid == 'NotExists':\n","      plid = createDWrecoverypl(target_ws_id,recovery_pipeline_prefix+'_'+source_ws + '_'+row['Warehouse Name'])\n","      if p_logging_verbose:\n","          print('Recovery pipeline ' + recovery_pipeline + ' created with Id '+plid)\n","    else:\n","      if p_logging_verbose:\n","          print('Datawarehouse recovery pipeline \"' + recovery_pipeline + '\" ('+plid+') already exist in workspace \"'+target_ws + '\" ('+target_ws_id+')')  \n","          print('\\n')\n","\n","    tablesToCopyParam = table_list[['schema','name']].to_json( orient='records')\n","    # ensure the temporary lakehouse exists\n","\n","    # obtain the connection string for the lakehouse to pass to the copy pipeline\n","    whurl  = \"v1/workspaces/\" + target_ws_id + \"/lakehouses/\" + temp_lh_id\n","    whresponse = client.get(whurl)\n","    lhconnStr = whresponse.json()['properties']['sqlEndpointProperties']['connectionString']\n","\n","    # get the SQLEndpoint ID of the lakehouse to pass to the copy pipeline\n","    items = fabric.list_items(workspace=target_ws_id)\n","    print(items)\n","    temp_lh_sqle_id = items[(items['Type'] == 'SQLEndpoint') & (items['Display Name']==lhname)]['Id'].values[0]\n","\n","\n","    # obtain the connection string for the warehouse to pass to the copy pipeline    \n","    whurl  = \"v1/workspaces/\" + target_ws_id + \"/warehouses/\" + target_wh_id\n","    whresponse = client.get(whurl)\n","    whconnStr = whresponse.json()['properties']['connectionInfo']\n","\n","    # obtain the pipeline id created to recover this warehouse\n","    plid = getItemId( target_ws_id,recovery_pipeline,'DataPipeline')\n","    if plid == 'NotExists':\n","        print('Error: Could not execute pipeline '+recovery_pipeline+ ' as the ID could not be obtained ')\n","    else:\n","        # pipeline url including pipeline Id unique to each warehouse\n","        plurl = 'v1/workspaces/'+target_ws_id+'/items/'+plid+'/jobs/instances?jobType=Pipeline'\n","        #print(plurl)\n","\n","        payload_data = '{' \\\n","            '\"executionData\": {' \\\n","                '\"parameters\": {' \\\n","                    '\"lakehouseId\": \"' + temp_lh_sqle_id + '\",' \\\n","                    '\"tablesToCopy\": ' + tablesToCopyParam + ',' \\\n","                    '\"workspaceId\": \"' + target_ws_id +'\",' \\\n","                    '\"warehouseId\": \"' + target_wh_id + '\",' \\\n","                    '\"lakehouseConnStr\": \"' + lhconnStr + '\",' \\\n","                    '\"warehouseConnStr\": \"' + whconnStr + '\"' \\\n","                    '}}}'\n","        #print(payload_data)\n","        plresponse = client.post(plurl, json=json.loads(payload_data))\n","        if p_logging_verbose:\n","            print(str(plresponse.status_code))      \n","print('Done')\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"collapsed":false},"id":"57dafef7-17a2-475f-9e62-eecc6660440c"},{"cell_type":"markdown","source":["##### Update directlake model lakehouse/warehouse connection\n","\n","https://semantic-link-labs.readthedocs.io/en/stable/sempy_labs.directlake.html#sempy_labs.directlake.update_direct_lake_model_connection    "],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"cc97be77-116e-4cde-bdc6-2971ab98a083"},{"cell_type":"code","source":["\n","df_datasets = fabric.list_datasets(target_ws)\n","\n","# Iterate over each dataset in the dataframe\n","for index, row in df_datasets.iterrows():\n","    # Check if the dataset is not the default semantic model\n","    if not labs.is_default_semantic_model(row['Dataset Name'], fabric.resolve_workspace_id(target_ws)):\n","        print('Updating semantic model connection ' + row['Dataset Name'] + ' in workspace '+ target_ws)\n","        labs.directlake.update_direct_lake_model_connection(dataset=row['Dataset Name'], \n","                                                                        workspace= target_ws,\n","                                                                        source=labs.directlake.get_direct_lake_source(row['Dataset Name'], workspace= target_ws)[1], \n","                                                                        source_type=labs.directlake.get_direct_lake_source(row['Dataset Name'], workspace= target_ws)[0], \n","                                                                        source_workspace=target_ws)\n","        labs.refresh_semantic_model(dataset=row['Dataset Name'], workspace= target_ws)\n","\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"9deccda6-5c3d-4b88-8ed8-68855ca0949a"},{"cell_type":"markdown","source":["##### Rebind reports to local datasets\n","\n","https://semantic-link-labs.readthedocs.io/en/latest/sempy_labs.report.html#sempy_labs.report.report_rebind"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"36783f3b-4904-4d74-842d-dbd026a3184a"},{"cell_type":"code","source":["df_reports = fabric.list_reports(workspace=target_ws)\n","for index, row in df_reports.iterrows():\n","    #print(row['Name'] + '-' + row['Dataset Id'])\n","    df_datasets = fabric.list_datasets(workspace=target_ws)\n","    dataset_name = df_datasets[df_datasets['Dataset ID'] == row['Dataset Id']]['Dataset Name'].values[0]\n","    print(f'Rebinding report to {dataset_name} in {target_ws}')\n","    labs.report.report_rebind(report=row['Name'],dataset=dataset_name, report_workspace=target_ws, dataset_workspace=target_ws)\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"},"collapsed":false},"id":"06268ede-b795-493e-9a8d-772654ce7e20"},{"cell_type":"markdown","source":["##### Update data pipeline source & sink connections\n","\n","Support changes lakehouses, warehouses, notebooks and connections from source to target. <br>\n","Connections changes should be expressed as an array of tuples [{from_1:to_1},{from_N:to_N}]"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"4ae65012-350c-40c0-a68a-4069c567a85f"},{"cell_type":"code","source":["from typing import Optional\n","from sempy_labs._helper_functions import (\n","    resolve_workspace_name_and_id,\n","    lro,\n","    _decode_b64,\n",")\n","import sempy_labs._icons as icons\n","\n","import base64\n","from typing import Optional, Tuple, List\n","from uuid import UUID\n","\n","\n","def update_data_pipeline_definition(\n","    name: str, pipeline_content: dict, workspace: Optional[str] = None\n","):\n","    \"\"\"\n","    Updates an existing data pipeline with a new definition.\n","\n","    Parameters\n","    ----------\n","    name : str\n","        The name of the data pipeline.\n","    pipeline_content : dict\n","        The data pipeline content (not in Base64 format).\n","    workspace : str, default=None\n","        The name of the workspace.\n","        Defaults to None which resolves to the workspace of the attached lakehouse\n","        or if no lakehouse attached, resolves to the workspace of the notebook.\n","    \"\"\"\n","\n","    (workspace, workspace_id) = resolve_workspace_name_and_id(workspace)\n","    client = fabric.FabricRestClient()\n","    pipeline_payload = base64.b64encode(json.dumps(pipeline_content).encode('utf-8')).decode('utf-8')\n","    pipeline_id = fabric.resolve_item_id(\n","        item_name=name, type=\"DataPipeline\", workspace=workspace\n","    )\n","\n","    request_body = {\n","        \"definition\": {\n","            \"parts\": [\n","                {\n","                    \"path\": \"pipeline-content.json\",\n","                    \"payload\": pipeline_payload,\n","                    \"payloadType\": \"InlineBase64\"\n","                }\n","            ]\n","        }\n","    }\n","\n","\n","    response = client.post(\n","        f\"v1/workspaces/{workspace_id}/items/{pipeline_id}/updateDefinition\",\n","        json=request_body,\n","    )\n","\n","    lro(client, response, return_status_code=True)\n","\n","    print(\n","        f\"{icons.green_dot} The '{name}' pipeline was updated within the '{workspace}' workspace.\"\n","    )\n","\n","def _is_valid_uuid(\n","    guid: str,\n","):\n","    \"\"\"\n","    Validates if a string is a valid GUID in version 4\n","\n","    Parameters\n","    ----------\n","    guid : str\n","        GUID to be validated.\n","\n","    Returns\n","    -------\n","    bool\n","        Boolean that indicates if the string is a GUID or not.\n","    \"\"\"\n","\n","    try:\n","        UUID(str(guid), version=4)\n","        return True\n","    except ValueError:\n","        return False"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"bdd46d4a-ef58-4f9a-b2e8-a428361a17c1"},{"cell_type":"code","source":["import json\n","from jsonpath_ng import jsonpath, parse\n","from typing import Optional, Tuple, List\n","from uuid import UUID\n","\n","source_ws = ''\n","target_ws = ''\n","\n","\n","# Swaps the connection properties of an activity belonging to the specified item type(s)\n","def swap_pipeline_connection(pl_json: dict, p_source_ws: str,p_target_ws: str, \n","                                p_item_type: List =['DataWarehouse','Lakehouse','Notebook'], \n","                                p_conn_id_from_to: Optional[List[Tuple[str,str]]]=[]):\n","    \n","    source_ws_id = fabric.resolve_workspace_id(source_ws)\n","\n","    target_ws_id = fabric.resolve_workspace_id(target_ws)\n","\n","    if 'Warehouse' in p_item_type or 'Lakehouse' in p_item_type:\n","        ls_expr = parse('$..linkedService')\n","        for endpoint_match in ls_expr.find(pl_json):\n","            if endpoint_match.value['properties']['type'] == 'DataWarehouse' \\\n","                and endpoint_match.value['properties']['typeProperties']['workspaceId'] == source_ws_id \\\n","                and 'Warehouse' in p_item_type:\n","                # only update the warehouse if it was located in the source workspace i.e. we will update the properties to the target workspace if the warehouse resided in the same workspace as the pipeline\n","                #print(endpoint_match.value)\n","                warehouse_id = endpoint_match.value['properties']['typeProperties']['artifactId']\n","                #print(warehouse_id)\n","                warehouse_endpoint = endpoint_match.value['properties']['typeProperties']['endpoint']\n","                #print(warehouse_endpoint)\n","                \n","                source_wh_name = fabric.resolve_item_name(item_id = warehouse_id,workspace=source_ws_id)\n","                #print(remote_wh_name)\n","                # find the warehouse id of the warehouse with the same name in the target workspace\n","                target_wh_id = fabric.resolve_item_id(item_name = source_wh_name,type='Warehouse',workspace=target_ws_id)\n","                # look up the connection string for the warehouse in the target workspace\n","                whurl  = f\"v1/workspaces/{target_ws_id}/warehouses/{target_wh_id}\"\n","                whresponse = client.get(whurl)\n","                lhconnStr = whresponse.json()['properties']['connectionString']\n","                endpoint_match.value['properties']['typeProperties']['artifactId'] = target_wh_id\n","                endpoint_match.value['properties']['typeProperties']['workspaceId'] = target_ws_id\n","                endpoint_match.value['properties']['typeProperties']['endpoint'] = lhconnStr\n","                #print(endpoint_match.value)\n","                ls_expr.update(endpoint_match,endpoint_match.value)\n","            if endpoint_match.value['properties']['type'] == 'Lakehouse' \\\n","                and endpoint_match.value['properties']['typeProperties']['workspaceId'] == source_ws_id \\\n","                and 'Lakehouse' in p_item_type:\n","                #print(endpoint_match.value)\n","                lakehouse_id = endpoint_match.value['properties']['typeProperties']['artifactId']\n","                remote_lh_name = fabric.resolve_item_name(item_id = lakehouse_id,workspace=source_ws_id)\n","                # find the lakehouse id of the lakehouse with the same name in the target workspace\n","                target_lh_id = fabric.resolve_item_id(item_name = remote_lh_name,type='Lakehouse',workspace=target_ws_id)\n","                endpoint_match.value['properties']['typeProperties']['artifactId'] = target_lh_id\n","                endpoint_match.value['properties']['typeProperties']['workspaceId'] = target_ws_id\n","                ls_expr.update(endpoint_match,endpoint_match.value)\n","                #    print(endpoint_match.value)\n","\n","\n","    if 'Notebook' in p_item_type: \n","        ls_expr = parse('$..activities')\n","\n","        for endpoint_match in ls_expr.find(pl_json):\n","            for activity in endpoint_match.value:\n","                #print(activity['type'])\n","                if activity['type']=='TridentNotebook' and 'Notebook' in p_item_type: #only update if the notebook was in the same workspace as the pipeline\n","                    print('change from '+activity['typeProperties']['workspaceId'])\n","                    source_nb_id = activity['typeProperties']['notebookId']\n","                    source_nb_name = fabric.resolve_item_name(item_id = source_nb_id,workspace=source_ws_id)\n","                    target_nb_id = fabric.resolve_item_id(item_name = source_nb_name,type='Notebook',workspace=target_ws_id)\n","                    activity['typeProperties']['notebookId']=target_nb_id\n","                    activity['typeProperties']['workspaceId']=target_ws_id\n","                    print('to notebook '+ target_nb_id)\n","                    #ls_expr.update(endpoint_match,endpoint_match.value)\n","\n","    if p_conn_from_to:\n","        for ti_conn_from_to in p_conn_from_to:\n","            if not _is_valid_uuid(ti_conn_from_to[0]):\n","                print('Connection from is string '+ str(ti_conn_from_to[0]))\n","                dfC_filt = df_conns[df_conns[\"Connection Name\"] == ti_conn_from_to[0]]       \n","                connId_from = dfC_filt['Connection Id'].iloc[0]     \n","            else:\n","                connId_from = ti_conn_from_to[0]\n","\n","            if not _is_valid_uuid(ti_conn_from_to[1]):\n","                print('Connection from is string '+ str(ti_conn_from_to[1]))\n","                dfC_filt = df_conns[df_conns[\"Connection Name\"] == ti_conn_from_to[1]]       \n","                connId_to = dfC_filt['Connection Id'].iloc[0]     \n","            else:\n","                connId_to = ti_conn_from_to[1]\n","\n","            ls_expr = parse('$..externalReferences')\n","            for externalRef in ls_expr.find(pl_json):\n","                if externalRef.value['connection']==connId_from:\n","                    print('Changing connection from '+str(connId_from))\n","                    externalRef.value['connection']=connId_to\n","                    ls_expr.update(externalRef,externalRef.value)\n","                    print('to '+str(connId_to))\n","\n","    return pl_json\n","\n","\n","\n","# loading a dataframe of connections to perform an ID lookup if required \n","df_conns = labs.list_connections()\n","\n","df_pipeline = labs.list_data_pipelines(target_ws)\n","for index, row in df_pipeline.iterrows():\n","    #print(labs.get_data_pipeline_definition(row['Data Pipeline Name'],target_ws))\n","    if row['Data Pipeline Name']=='plRecover_WH6_Prod2_Warehouse2_fixed':\n","        pipeline_json = json.loads(labs.get_data_pipeline_definition(row['Data Pipeline Name'],source_ws))\n","\n","        p_new_json = swap_pipeline_connection(pipeline_json, source_ws,target_ws,\n","                ['DataWarehouse','Lakehouse','Notebook'],\n","                [p_connections_from_to]) \n","        #print(json.dumps(pipeline_json, indent=4))\n","        \n","        update_data_pipeline_definition(name=row['Data Pipeline Name'],pipeline_content=pipeline_json, workspace=target_ws)\n"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"079958e8-2880-484a-a994-41caf47e747e"},{"cell_type":"markdown","source":["##### Commit changes made above to Git"],"metadata":{"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"44174276-b983-4e80-9451-0afb9589cf1f"},{"cell_type":"code","source":["labs.commit_to_git(comment='Initial',  workspace=target_ws)"],"outputs":[],"execution_count":null,"metadata":{"jupyter":{"source_hidden":false,"outputs_hidden":false},"nteract":{"transient":{"deleting":false}},"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"9a5c3d84-f71d-4348-b419-c4953ac9e1d0"}],"metadata":{"kernel_info":{"name":"synapse_pyspark"},"kernelspec":{"name":"synapse_pyspark","language":"Python","display_name":"Synapse PySpark"},"language_info":{"name":"python"},"microsoft":{"language":"python","language_group":"synapse_pyspark","ms_spell_check":{"ms_spell_check_language":"en"}},"widgets":{},"nteract":{"version":"nteract-front-end@1.0.0"},"synapse_widget":{"version":"0.1","state":{}},"spark_compute":{"compute_id":"/trident/default","session_options":{"conf":{"spark.synapse.nbs.session.timeout":"1200000"}}},"dependencies":{"lakehouse":{}}},"nbformat":4,"nbformat_minor":5}
\ No newline at end of file