Add files via upload

Author: stephbruno

Deploy-PPE-PBIModel.cs

@@ -0,0 +1,30 @@
+//add a value pair for each parameter name and parameter value. 
+//For values with quotes, use the escape character \ before the quote.
+string[,] a = new string[,] {
+     {"serverName","\"ppeserver.database.windows.net\""}, //set the config for each env
+     {"databaseName","\"DataHub\""}, //ContosoPPEDb, ContosoProdDb
+     {"StartDate", "#date(2017, 1, 1)"}, //#datetime(2022, 5, 1, 0, 0, 0), #datetime(2017, 1, 1, 0, 0, 0)
+};
+
+//DO NOT CHANGE ANYTHING BELOW THIS LINE
+string paramExpression;
+int paramMetaLoc;
+string oldParameterVal;
+string newParameterVal;
+string newParamExpression;
+int x, y;
+foreach(var e in Model.Expressions)
+{
+    paramExpression = Model.Expressions[e.Name].Expression;
+    paramMetaLoc = paramExpression.IndexOf(" meta [IsParameterQuery=true");
+    if (paramMetaLoc > 0) {     //only do the rest if this is a parameter type query
+        for (x = 0; x < a.GetLength(0); x++) {
+            if (e.Name == a[x,0]) { //match up the parameters in the model with those defined above
+                oldParameterVal = paramExpression.Left(paramMetaLoc);
+                newParameterVal = a[x,1];
+                newParamExpression = newParameterVal + paramExpression.Substring(paramMetaLoc);
+                Model.Expressions[e.Name].Expression = newParamExpression;  //update the parameter
+            }
+        }
+    }
+}

Deploy-PPE-PBIModel.yml

@@ -0,0 +1,21 @@
+# Tabular Editor: https://docs.tabulareditor.com/te2/Command-line-Options.html
+
+trigger:
+  branches:
+    include:
+      - main                                                                              # the name of the branch that should trigger the pipeline
+  paths:
+    include:
+      - DatasetPath/                                                                      # the pipeline auto triggers when there is a change commited to branch
+
+variables:
+  DataSource: powerbi://api.powerbi.com/v1.0/myorg/Workspace Name                         # the Workspace where you will deploy
+  DeploymentDB: TestDataset                                                               # the name of the dataset to deploy to
+  UserId: app:xxxxxx-xxxxxx-xxxxxx-xxxxxx-xxxxxxxx@yyyyyy-yyyyyy-yyyyyy-yyyyyy-yyyyyyyy   # the UserId that has access to the workspace to deploy and refresh (in the form of app:AppId@TenantId)
+  AppId: xxxxxx-xxxxxx-xxxxxx-xxxxxx-xxxxxxxx                                             # the Service Principal identifier
+  TenantId: yyyyyy-yyyyyy-yyyyyy-yyyyyy-yyyyyyyy                                          # the tenant where the service prinicipal is registered
+  SourceControlPath: DatasetPath                                                          # the path in ADO where your Tabular model checked in as code.
+  ScriptFilePath: Pipelines\Deploy-PPE-PBIModel.cs                                        # the path for the script that Tabular Editor should run before deployment. Sample updates parameters.
+
+jobs:
+- template: PBI-Model-Deploy-Template.yml                                                 # the name of the yaml template file

PBI-Model-Deploy-Template.yml

@@ -0,0 +1,77 @@
+jobs:
+- job: AgentJob
+  displayName: Deploy Tabular Model
+  pool:
+    vmImage: 'windows-latest'
+  steps:
+  - checkout: self
+
+  - task: PowerShell@2
+    displayName: 1. Download Tabular Editor
+    inputs:
+      targetType: inline
+      script: "# Download URL for Tabular Editor portable: \n\n$TabularEditorUrl = \"https://github.com/otykier/TabularEditor/releases/download/2.16.5/TabularEditor.Portable.zip\"\n\n# Download destination (root of PowerShell script execution path): \n$DownloadDestination = join-path (get-location) \"TabularEditor.zip\" \necho($DownloadDestination)\n\n# Download from GitHub: \nInvoke-WebRequest -Uri $TabularEditorUrl -OutFile $DownloadDestination \n\n \n# Unzip Tabular Editor portable, and then delete the zip file:\n \nExpand-Archive -Path $DownloadDestination -DestinationPath (get-location).Path \n#Remove-Item $DownloadDestination "
+
+  - task: CmdLine@2
+    displayName: 2. Best Practice Check
+    inputs:
+      script: TabularEditor.exe "$(SourceControlPath)\database.json" -A -T testbpa.trx -V
+
+  - task: CopyFiles@2
+    displayName: 3. Copy Source Files
+    inputs:
+      SourceFolder: $(SourceControlPath)
+      Contents: database.json
+      TargetFolder: $(Build.ArtifactStagingDirectory)/PBIModel/Scripts
+
+  - task: CmdLine@2
+    displayName: 4. Schema Check
+    inputs:
+      script: TabularEditor.exe "$(Build.ArtifactStagingDirectory)\PBIModel\Scripts\database.json" -S "Pipelines\SetCredentialsForValidation.csx" -SC -T testsc.trx -V
+
+  - task: AzureKeyVault@2
+    displayName: 5. Read Key vault
+    inputs:
+      connectedServiceName: 'MyServiceConnectionName'   # The name of the Service Connection to access the key vault
+      KeyVaultName: 'MyKeyVauleName'                    # The name of the key vault
+      SecretsFilter: 'mysecret'                         # This step copies the password to this variable, so in subseqent steps the name should match. If you change it here, do a find and replace to change it everywhere in this file.
+      RunAsPreJob: true
+
+  - task: CmdLine@2
+    displayName: 6. Deployment Check
+    inputs:
+      script: TabularEditor.exe "$(Build.ArtifactStagingDirectory)\PBIModel\Scripts\database.json" -D "Provider=MSOLAP;Data Source=$(DataSource);User ID=$(UserId);password=$(mysecret)" "$(DeploymentDB)" -O -C -E -W -T testdep.trx -V
+
+  - task: CmdLine@2
+    displayName: 7. Create BIM File
+    inputs:
+      script: TabularEditor.exe "$(Build.SourcesDirectory)\$(SourceControlPath)" -BIM "$(Build.ArtifactStagingDirectory)\Model.bim"
+
+  - task: PublishBuildArtifacts@1
+    displayName: 8. Publish Artifact DBArtifacts
+    inputs:
+      PathtoPublish: $(build.ArtifactStagingDirectory)
+      ArtifactName: DBArtifacts
+
+  - task: CmdLine@2
+    displayName: 9. Deploy BIM File
+    inputs:
+      script: TabularEditor.exe "$(Build.ArtifactStagingDirectory)\Model.bim" -S "$(ScriptFilePath)" -D "Provider=MSOLAP;Data Source=$(DataSource);User ID=$(UserId);password=$(mysecret)" "$(DeploymentDB)" -O -P -Y
+
+  - task: PowerShell@2
+    displayName: '10. Refresh Power BI dataset'
+    inputs:
+      targetType: 'inline'
+      script: |
+        Install-Module -Name MicrosoftPowerBIMgmt -Scope CurrentUser -Force
+        $SecurePassword = ConvertTo-SecureString “$(mysecret)” -AsPlainText -Force  
+        $credential = [System.Management.Automation.PSCredential]::new("$(AppId)", $SecurePassword)  
+        Connect-PowerBIServiceAccount -ServicePrincipal -Tenant $(TenantId) -Credential $credential
+        $WorkspaceName = "$(DataSource)".split("/")[-1]
+        $Workspace = Get-PowerBIWorkspace -Name $WorkspaceName
+        $groupId = $Workspace[0].Id
+        $Dataset = Get-PowerBIDataset -WorkspaceId $groupId | Where-Object {$_.name -eq "$(DeploymentDB)"}
+        $datasetId = $Dataset[0].Id
+
+        Invoke-PowerBIRestMethod -Url "https://api.powerbi.com/v1.0/myorg/groups/$groupId/datasets/$datasetId/refreshes" -Method POST  -WarningAction Ignore
+

SetCredentialsForValidation.csx

@@ -0,0 +1,34 @@
+// This script replaces all Power Query partitions on this model with a
+// The script assumes that all Power Query partitions load data from the same SQL Server-based data source.
+
+// This function will extract all quoted values from the M expression, returning a list of strings
+// with the values extracted (in order), but ignoring any quoted values where a hashtag (#) precedes
+// the quotation mark:
+var split = new Func<string, List<string>>(m => {
+    var result = new List<string>();
+    var i = 0;
+    foreach (var s in m.Split('"'))
+    {
+        if (s.EndsWith("#") && i % 2 == 0) i = -2;
+        if (i >= 0 && i % 2 == 1) result.Add(s);
+        i++;
+    }
+    return result;
+});
+var GetServer = new Func<string, string>(m => split(m)[0]);    // Server name is usually the 1st encountered string
+var GetDatabase = new Func<string, string>(m => split(m)[1]);  // Database name is usually the 2nd encountered string
+var GetSchema = new Func<string, string>(m => split(m)[2]);    // Schema name is usually the 3rd encountered string
+var GetTable = new Func<string, string>(m => split(m)[3]);     // Table name is usually the 4th encountered string
+
+// Remove Power Query partitions from all tables and replace them with a single Legacy partition:
+foreach (var t in Model.Tables.Where(t => t.Partitions.OfType<MPartition>().Any()))
+{
+    var mPartitions = t.Partitions.OfType<MPartition>();
+    if (!mPartitions.Any()) continue;
+
+    var schema = GetSchema(mPartitions.First().Expression);
+    var table = GetTable(mPartitions.First().Expression);
+
+    t.AddPartition(t.Name, string.Format("SELECT * FROM [{0}].[{1}]", schema, table));
+    foreach (var p in mPartitions.ToList()) p.Delete();
+}