Skip to content

Commit bf30690

Browse files
AzureNickAzureNick
authored
Merge pull request #40 from MarkPryceMaherMSFT/main
Initial check in
2 parents 81c508e + 7b6a874 commit bf30690

10 files changed

+858
-0
lines changed

‎tools/Gen2toFabricDW/Config.cs

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Text;
5+
using System.Threading.Tasks;
6+
7+
namespace Gen2toFabricDW
8+
{
9+
public class Config
10+
{
11+
public string Gen2Connectionstring { get; set; }
12+
public string fabricEndpoint { get; set; }
13+
public string fabricWarehouse{ get; set; }
14+
public List<Table> Tables { get; set; }
15+
public string SQLCreateExternalTable { get; set; }
16+
public string COPYINTO_Statement { get; set; }
17+
public int batchsize { get; set; }
18+
public string SPN_Application_ID { get; set; }
19+
public string SPN_Secret { get; set; }
20+
public string SPN_Tenant { get; set; }
21+
public string abfslocation { get; set; }
22+
public string httpslocation { get; set; }
23+
public string SASKey { get; set; }
24+
25+
}
26+
27+
public class Table
28+
{
29+
public string Name { get; set; }
30+
public string Schema { get; set; }
31+
public string DropDestinationTable { get; set; }
32+
public string CreateDestination { get; set; }
33+
public string TruncateDestination { get; set; }
34+
public string batchcolumn { get; set; }
35+
public string externalTableSchema { get; set; }
36+
}
37+
}

‎tools/Gen2toFabricDW/ExternalTableSetup.ssql

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
if not exists(select * from sys.external_file_formats where name='_extract_parquet')
2+
begin
3+
CREATE EXTERNAL FILE FORMAT _extract_parquet
4+
WITH ( FORMAT_TYPE = PARQUET, DATA_COMPRESSION = 'org.apache.hadoop.io.compress.SnappyCodec')
5+
end
6+
7+
--DROP DATABASE SCOPED CREDENTIAL [_extract_cred]
8+
if not exists(SELECT name, create_date, modify_date FROM sys.database_scoped_credentials where name ='_extract_cred')
9+
BEGIN
10+
CREATE DATABASE SCOPED CREDENTIAL [_extract_cred] WITH IDENTITY='SHARED ACCESS SIGNATURE', SECRET = '{SASKey}'
11+
END
12+
13+
-- drop EXTERNAL DATA SOURCE [_extract_storage]
14+
if not exists(SELECT name FROM sys.external_data_sources where name = '_extract_storage')
15+
BEGIN
16+
CREATE EXTERNAL DATA SOURCE [_extract_storage] WITH ( type=hadoop, LOCATION = N'{abfslocation}', CREDENTIAL = [_extract_cred])
17+
END
18+
19+
/*
20+
-- script to list all external tables
21+
select 'drop external table [' + s.name + '].[' + t.name + ']' from sys.tables t inner join sys.schemas s on s.schema_id = t.schema_id
22+
where t.is_external = 1
23+
*/
24+

‎tools/Gen2toFabricDW/Gen2toFabricDW.csproj

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net8.0</TargetFramework>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
</PropertyGroup>
9+
10+
<ItemGroup>
11+
<PackageReference Include="Microsoft.Data.SqlClient" Version="6.0.1" />
12+
</ItemGroup>
13+
14+
<ItemGroup>
15+
<None Update="config.json">
16+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
17+
</None>
18+
<None Update="generatetableschema.ssql">
19+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
20+
</None>
21+
<None Update="ExternalTableSetup.ssql">
22+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
23+
</None>
24+
</ItemGroup>
25+
26+
</Project>

‎tools/Gen2toFabricDW/Gen2toFabricDW.sln

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
2+
Microsoft Visual Studio Solution File, Format Version 12.00
3+
# Visual Studio Version 17
4+
VisualStudioVersion = 17.12.35707.178 d17.12
5+
MinimumVisualStudioVersion = 10.0.40219.1
6+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Gen2toFabricDW", "Gen2toFabricDW.csproj", "{7A83E260-A325-4696-9D4D-9B8BCE75256C}"
7+
EndProject
8+
Global
9+
GlobalSection(SolutionConfigurationPlatforms) = preSolution
10+
Debug|Any CPU = Debug|Any CPU
11+
Release|Any CPU = Release|Any CPU
12+
EndGlobalSection
13+
GlobalSection(ProjectConfigurationPlatforms) = postSolution
14+
{7A83E260-A325-4696-9D4D-9B8BCE75256C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
15+
{7A83E260-A325-4696-9D4D-9B8BCE75256C}.Debug|Any CPU.Build.0 = Debug|Any CPU
16+
{7A83E260-A325-4696-9D4D-9B8BCE75256C}.Release|Any CPU.ActiveCfg = Release|Any CPU
17+
{7A83E260-A325-4696-9D4D-9B8BCE75256C}.Release|Any CPU.Build.0 = Release|Any CPU
18+
EndGlobalSection
19+
GlobalSection(SolutionProperties) = preSolution
20+
HideSolutionNode = FALSE
21+
EndGlobalSection
22+
EndGlobal

‎tools/Gen2toFabricDW/Logging.cs

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Text;
5+
using System.Threading.Tasks;
6+
7+
namespace Gen2toFabricDW
8+
{
9+
public static class Logging
10+
{
11+
public static string locallogging = string.Empty;
12+
public static void Log(string message, string logLevel = "INFO")
13+
{
14+
string logMessage = $"{DateTime.Now:yyyy-MM-dd HH:mm:ss} [{logLevel}] {message}";
15+
Console.WriteLine(logMessage);
16+
try
17+
{
18+
File.AppendAllText(locallogging, logMessage + Environment.NewLine);
19+
}
20+
catch (Exception ex)
21+
{
22+
Console.WriteLine($"Failed to write log: {ex.Message}");
23+
}
24+
}
25+
26+
}
27+
}

0 commit comments

Comments
 (0)