Initial check in

tools/Gen2toFabricDW/Config.cs

@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Gen2toFabricDW
+{
+    public class Config
+    {
+        public string Gen2Connectionstring { get; set; }
+        public string fabricEndpoint { get; set; }
+        public string fabricWarehouse{ get; set; }
+        public List<Table> Tables { get; set; } 
+        public string SQLCreateExternalTable { get; set; }
+        public string COPYINTO_Statement { get; set; }
+        public int batchsize { get; set; }
+        public string SPN_Application_ID { get; set; }
+        public string SPN_Secret { get; set; }
+        public string SPN_Tenant { get; set; }
+        public string abfslocation { get; set; }
+        public string httpslocation { get; set; }
+        public string SASKey { get; set; }
+
+    }
+
+    public class Table
+    {
+        public string Name { get; set; }
+        public string Schema { get; set; }
+        public string DropDestinationTable { get; set; }
+        public string CreateDestination { get; set; }
+        public string TruncateDestination { get; set; }
+        public string batchcolumn { get; set; }
+        public string externalTableSchema { get; set; }
+    }
+}

tools/Gen2toFabricDW/ExternalTableSetup.ssql

@@ -0,0 +1,24 @@
+if not exists(select * from sys.external_file_formats where name='_extract_parquet')
+begin
+	CREATE EXTERNAL FILE FORMAT _extract_parquet  
+	WITH (      FORMAT_TYPE = PARQUET,    DATA_COMPRESSION = 'org.apache.hadoop.io.compress.SnappyCodec')
+end
+
+--DROP DATABASE SCOPED CREDENTIAL [_extract_cred]
+if not exists(SELECT name, create_date, modify_date  FROM sys.database_scoped_credentials where name ='_extract_cred')
+BEGIN
+	CREATE DATABASE SCOPED CREDENTIAL [_extract_cred] WITH IDENTITY='SHARED ACCESS SIGNATURE',   SECRET = '{SASKey}'
+END
+
+-- drop EXTERNAL DATA SOURCE [_extract_storage]
+if not exists(SELECT name   FROM sys.external_data_sources where name = '_extract_storage')
+BEGIN
+	CREATE EXTERNAL DATA SOURCE [_extract_storage] WITH ( type=hadoop, LOCATION = N'{abfslocation}', CREDENTIAL = [_extract_cred])
+END
+
+/*
+-- script to list all external tables
+select 'drop external table [' + s.name + '].[' +  t.name + ']'  from sys.tables t inner join sys.schemas s on s.schema_id = t.schema_id 
+where t.is_external = 1
+*/
+

tools/Gen2toFabricDW/Gen2toFabricDW.csproj

@@ -0,0 +1,26 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Data.SqlClient" Version="6.0.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="config.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="generatetableschema.ssql">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="ExternalTableSetup.ssql">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

tools/Gen2toFabricDW/Gen2toFabricDW.sln

@@ -0,0 +1,22 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.12.35707.178 d17.12
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Gen2toFabricDW", "Gen2toFabricDW.csproj", "{7A83E260-A325-4696-9D4D-9B8BCE75256C}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{7A83E260-A325-4696-9D4D-9B8BCE75256C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7A83E260-A325-4696-9D4D-9B8BCE75256C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7A83E260-A325-4696-9D4D-9B8BCE75256C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7A83E260-A325-4696-9D4D-9B8BCE75256C}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

tools/Gen2toFabricDW/Logging.cs

@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Gen2toFabricDW
+{
+    public static class Logging
+    {
+        public static string locallogging = string.Empty;
+        public static void Log(string message, string logLevel = "INFO")
+        {
+            string logMessage = $"{DateTime.Now:yyyy-MM-dd HH:mm:ss} [{logLevel}] {message}";
+            Console.WriteLine(logMessage);
+            try
+            {
+                File.AppendAllText(locallogging, logMessage + Environment.NewLine);
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Failed to write log: {ex.Message}");
+            }
+        }
+
+    }
+}