From 8ac5a9e2c357f514702d9f857c48e67d55d5f59e Mon Sep 17 00:00:00 2001 From: Adam Hopkins <127156771+simpat-adam@users.noreply.github.com> Date: Thu, 7 Dec 2023 12:53:56 -0600 Subject: [PATCH] [DI-1132] Add Delete By Id Functionality (#47) * Add IsDeleteOperation to DataMap * Allow UI to set,edit IsDeleteOperation * Enable DataMap editing for delete by id use case * Updating DataImport exe. Successfully deleting by id * Simplify UI for use case * Delete by Id disclaimer * Naming and formatting * DataMapSerializerTests coverage of new serialize and deserialize paths * Update pkg with vulnerabiities * Fix param order * troubleshooting build * Fix python.exe process tests having to do with accessing process name after process has completed. * Add Web tests * use await on migrate commands * Try only one test project * Force test projects to run in series * Move delete serialization to dedicated class --- DataImport.Common.Tests/appsettings.json | 2 +- DataImport.Common/DataImport.Common.csproj | 2 +- .../ExternalPreprocessorService.cs | 2 +- .../DataMapSerializerTests.cs | 135 +- DataImport.Models/DataMapSerializer.cs | 2 +- DataImport.Models/Datamap.cs | 2 + DataImport.Models/DeleteDataMapSerializer.cs | 86 ++ ...3229_AddIsDeleteOperationField.Designer.cs | 1123 ++++++++++++++++ ...0231128143229_AddIsDeleteOperationField.cs | 26 + ...tgreSqlDataImportDbContextModelSnapshot.cs | 5 +- ...2509_AddIsDeleteOperationField.Designer.cs | 1125 +++++++++++++++++ ...0231127222509_AddIsDeleteOperationField.cs | 26 + .../SqlDataImportDbContextModelSnapshot.cs | 5 +- .../LoadResources/TestFailingOdsApi.cs | 5 + .../Features/LoadResources/TestOdsApi.cs | 22 + .../Features/LoadResources/FileProcessor.cs | 55 +- .../Features/LoadResources/OdsApi.cs | 31 + .../Features/LoadResources/ResourceMapper.cs | 26 +- .../Features/DataMaps/AddEditDataMapsTests.cs | 161 ++- DataImport.Web.Tests/SetUpFixture.cs | 6 +- .../Features/DataMaps/AddDataMap.cs | 8 +- .../Features/DataMaps/AddEdit.cshtml | 35 +- .../DataMaps/AddEditDataMapViewModel.cs | 3 + .../Features/DataMaps/DataMapperFields.cs | 8 +- .../Features/DataMaps/DataMapsController.cs | 8 +- .../Features/DataMaps/EditDataMap.cs | 19 +- .../DataMapperDeleteById.cshtml | 67 + .../_PartialDataMapperDeleteByIdFields.cshtml | 19 + DataImport.Web/Helpers/JsonValidator.cs | 2 +- build.ps1 | 4 +- 30 files changed, 2939 insertions(+), 81 deletions(-) create mode 100644 DataImport.Models/DeleteDataMapSerializer.cs create mode 100644 DataImport.Models/Migrations/PostgreSql/20231128143229_AddIsDeleteOperationField.Designer.cs create mode 100644 DataImport.Models/Migrations/PostgreSql/20231128143229_AddIsDeleteOperationField.cs create mode 100644 DataImport.Models/Migrations/SqlServer/20231127222509_AddIsDeleteOperationField.Designer.cs create mode 100644 DataImport.Models/Migrations/SqlServer/20231127222509_AddIsDeleteOperationField.cs create mode 100644 DataImport.Web/Features/DataMaps/EditorTemplates/DataMapperDeleteById.cshtml create mode 100644 DataImport.Web/Features/DataMaps/_PartialDataMapperDeleteByIdFields.cshtml diff --git a/DataImport.Common.Tests/appsettings.json b/DataImport.Common.Tests/appsettings.json index 9007e7aa..c6385d2f 100644 --- a/DataImport.Common.Tests/appsettings.json +++ b/DataImport.Common.Tests/appsettings.json @@ -1,3 +1,3 @@ -{ +{ "PythonExecutableLocation": "..\\..\\..\\..\\.tools\\python\\python.exe" } diff --git a/DataImport.Common/DataImport.Common.csproj b/DataImport.Common/DataImport.Common.csproj index 4472b103..cb250cd5 100644 --- a/DataImport.Common/DataImport.Common.csproj +++ b/DataImport.Common/DataImport.Common.csproj @@ -1,4 +1,4 @@ - + Library diff --git a/DataImport.Common/Preprocessors/ExternalPreprocessorService.cs b/DataImport.Common/Preprocessors/ExternalPreprocessorService.cs index 50def83e..96512d53 100644 --- a/DataImport.Common/Preprocessors/ExternalPreprocessorService.cs +++ b/DataImport.Common/Preprocessors/ExternalPreprocessorService.cs @@ -90,7 +90,7 @@ void WriteToCollection(DataReceivedEventArgs eventArgs, Collection colle { using var process = Process.Start(startInfo); - var processName = process.ProcessName; + var processName = process.HasExited ? "Exited python process" : process.ProcessName; _logger.LogInformation("External preprocess {ProcessName} started", processName); process.OutputDataReceived += (_, e) => { WriteToCollection(e, outputLines); }; diff --git a/DataImport.Models.Tests/DataMapSerializerTests.cs b/DataImport.Models.Tests/DataMapSerializerTests.cs index 697a1146..97752623 100644 --- a/DataImport.Models.Tests/DataMapSerializerTests.cs +++ b/DataImport.Models.Tests/DataMapSerializerTests.cs @@ -4,6 +4,7 @@ // See the LICENSE and NOTICES files in the project root for more information. using System; +using System.Linq; using NUnit.Framework; using DataImport.TestHelpers; using Newtonsoft.Json; @@ -135,8 +136,28 @@ public void ShouldSerializeAndDeserializeJsonMapRepresentationOfMappings() ] }"; - Serialize(_resourceMetadata, mappings).ShouldMatch(jsonMap); - Deserialize(_resourceMetadata, jsonMap).ShouldMatch(mappings); + SerializeNormalMap(_resourceMetadata, mappings).ShouldMatch(jsonMap); + DeserializeNormalMap(_resourceMetadata, jsonMap).ShouldMatch(mappings); + } + + [Test] + public void ShouldSerializeAndDeserializeJsonMapRepresentationOfDeleteByIdMappings() + { + //Deleting by Id is a unique case where the mapping will always be between exactly one source column + //and a property always called 'Id' which is not part of the underlying metadata. + var mappings = new[] + { + MapColumn("propertyB", "ColumnB"), + }; + + var jsonMap = @"{ + ""Id"": { + ""Column"": ""ColumnB"" + } + }"; + + SerializeDeleteByIdMap(mappings).ShouldMatch(jsonMap); + DeserializeDeleteByIdMap(_resourceMetadata, jsonMap).Single().SourceColumn.ShouldMatch(mappings.Single().SourceColumn); } [Test] @@ -149,7 +170,7 @@ public void ShouldSerializeToMinimalJsonMapWhenMappingsAreSubsetOfMetadata() //as the most natural behavior. var emptyMappings = new DataMapper[] { }; - Serialize(_resourceMetadata, emptyMappings).ShouldMatch("{}"); + SerializeNormalMap(_resourceMetadata, emptyMappings).ShouldMatch("{}"); var partialMappings = new[] { @@ -165,7 +186,7 @@ public void ShouldSerializeToMinimalJsonMapWhenMappingsAreSubsetOfMetadata() ""Column"": ""ColumnB"" } }"; - Serialize(_resourceMetadata, partialMappings).ShouldMatch(expectedJsonMap); + SerializeNormalMap(_resourceMetadata, partialMappings).ShouldMatch(expectedJsonMap); } [Test] @@ -217,8 +238,8 @@ public void ShouldSerializeMappingsOmittingUnmappedArrayElements() ] }"; - Serialize(resourceMetadata, mappingsIncludingUnmappedItem).ShouldMatch(jsonLackingUnmappedItem); - Deserialize(resourceMetadata, jsonLackingUnmappedItem).ShouldMatch(mappingsLackingUnmappedItem); + SerializeNormalMap(resourceMetadata, mappingsIncludingUnmappedItem).ShouldMatch(jsonLackingUnmappedItem); + DeserializeNormalMap(resourceMetadata, jsonLackingUnmappedItem).ShouldMatch(mappingsLackingUnmappedItem); } [Test] @@ -238,7 +259,7 @@ public void ShouldFailToSerializeMappingsWithKeysNotFoundInMetadata() MapColumn("unexpectedProperty", "ColumnZ") }; - Action attemptToSerializeInvalidMapping = () => Serialize(_resourceMetadata, invalidMappings); + Action attemptToSerializeInvalidMapping = () => SerializeNormalMap(_resourceMetadata, invalidMappings); attemptToSerializeInvalidMapping .ShouldThrow() @@ -252,7 +273,7 @@ public void ShouldFailToSerializeMappingsWhichMapSingleValueWhenExpectingAnObjec { Action attemptToSerializeStaticMappingToExpectedObject = () => { - Serialize(_resourceMetadata, new[] + SerializeNormalMap(_resourceMetadata, new[] { MapStatic("complexProperty", "Static Value"), }); @@ -267,7 +288,7 @@ public void ShouldFailToSerializeMappingsWhichMapSingleValueWhenExpectingAnObjec Action attemptToSerializeColumnMappingToExpectedObject = () => { - Serialize(_resourceMetadata, new[] + SerializeNormalMap(_resourceMetadata, new[] { MapColumn("complexProperty", "ColumnA"), }); @@ -286,7 +307,7 @@ public void ShouldFailToSerializeMappingsWhichMapSingleValueWhenExpectingAnArray { Action attemptToSerializeStaticMappingToExpectedArray = () => { - Serialize(_resourceMetadata, new[] + SerializeNormalMap(_resourceMetadata, new[] { MapStatic("arrayProperty", "Static Value"), }); @@ -301,7 +322,7 @@ public void ShouldFailToSerializeMappingsWhichMapSingleValueWhenExpectingAnArray Action attemptToSerializeColumnMappingToExpectedArray = () => { - Serialize(_resourceMetadata, new[] + SerializeNormalMap(_resourceMetadata, new[] { MapColumn("arrayProperty", "ColumnA"), }); @@ -320,7 +341,7 @@ public void ShouldFailToSerializeMappingsWhichMapComplexValuesWhenExpectingSingl { Action attemptToSerializeObjectMappingToExpectedSingleValue = () => { - Serialize(_resourceMetadata, new[] + SerializeNormalMap(_resourceMetadata, new[] { MapObject("propertyA", MapColumn("nestedPropertyF", "ColumnF"), @@ -338,7 +359,7 @@ public void ShouldFailToSerializeMappingsWhichMapComplexValuesWhenExpectingSingl Action attemptToSerializeArrayMappingToExpectedSingleValue = () => { - Serialize(_resourceMetadata, new[] + SerializeNormalMap(_resourceMetadata, new[] { MapArray( "propertyA", @@ -426,7 +447,24 @@ public void ShouldDeserializeFromPreviouslyParsedJObject() ] }"; - Deserialize(_resourceMetadata, JObject.Parse(jsonMap)).ShouldMatch(mappings); + DeserializeNormalMap(_resourceMetadata, JObject.Parse(jsonMap)).ShouldMatch(mappings); + } + + [Test] + public void ShouldDeserializeFromPreviouslyParsedDeleteByIdJObject() + { + var mappings = new[] + { + MapColumn("propertyB", "ColumnB") + }; + + var jsonMap = @"{ + ""Id"": { + ""Column"": ""ColumnB"" + } + }"; + + DeserializeDeleteByIdMap(JObject.Parse(jsonMap)).Single().SourceColumn.ShouldMatch(mappings.Single().SourceColumn); } [Test] @@ -440,7 +478,7 @@ public void ShouldDeserializeIncludingUnmappedPropertiesWhenJsonMapIsSubsetOfMet //in the JSON Map to deserialize is not meaningful, so the output is normalized to //metadata order as a result of ensuring completeness of the result - Deserialize(_resourceMetadata, "{}") + DeserializeNormalMap(_resourceMetadata, "{}") .ShouldMatch( Unmapped("propertyA"), Unmapped("propertyB"), @@ -472,7 +510,7 @@ public void ShouldDeserializeIncludingUnmappedPropertiesWhenJsonMapIsSubsetOfMet ""Column"": ""ColumnB"" } }"; - Deserialize(_resourceMetadata, partialJsonMap) + DeserializeNormalMap(_resourceMetadata, partialJsonMap) .ShouldMatch( Unmapped("propertyA"), MapColumn("propertyB", "ColumnB"), @@ -505,7 +543,7 @@ public void ShouldFailToDeserializeTextThatIsNotJson() var invalidJsonMap = @"This plain text is not JSON."; - Action attemptToDeserializeInvalidJsonMap = () => Deserialize(_resourceMetadata, invalidJsonMap); + Action attemptToDeserializeInvalidJsonMap = () => DeserializeNormalMap(_resourceMetadata, invalidJsonMap); var argumentException = attemptToDeserializeInvalidJsonMap .ShouldThrow(); @@ -540,7 +578,7 @@ public void ShouldFailToDeserializeJsonMapWithKeysNotFoundInMetadata() ""unexpectedProperty"": { ""Column"": ""ColumnZ"" } }"; - Action attemptToDeserializeInvalidJsonMap = () => Deserialize(_resourceMetadata, invalidJsonMap); + Action attemptToDeserializeInvalidJsonMap = () => DeserializeNormalMap(_resourceMetadata, invalidJsonMap); attemptToDeserializeInvalidJsonMap .ShouldThrow() @@ -554,7 +592,7 @@ public void ShouldFailToDeserializeJsonMapWithUnexpectedFormat() { //The outermost JSON should be an { object }, not an [ array ]. Action attemptToDeserializeInvalidTopLevelObject = - () => Deserialize(_resourceMetadata, @"[]"); + () => DeserializeNormalMap(_resourceMetadata, @"[]"); attemptToDeserializeInvalidTopLevelObject .ShouldThrow() .Message @@ -565,7 +603,7 @@ public void ShouldFailToDeserializeJsonMapWithUnexpectedFormat() //Metadata declares that "complexProperty" should be an { object }, not an [ array ]. Action attemptToDeserializeInvalidObjectValue = - () => Deserialize(_resourceMetadata, @"{ ""complexProperty"": [] }"); + () => DeserializeNormalMap(_resourceMetadata, @"{ ""complexProperty"": [] }"); attemptToDeserializeInvalidObjectValue .ShouldThrow() .Message @@ -575,7 +613,7 @@ public void ShouldFailToDeserializeJsonMapWithUnexpectedFormat() //Metadata declares that "arrayProperty" should be an [ array ], not a boolean. Action attemptToDeserializeInvalidArrayValue = - () => Deserialize(_resourceMetadata, @"{ ""arrayProperty"": true }"); + () => DeserializeNormalMap(_resourceMetadata, @"{ ""arrayProperty"": true }"); attemptToDeserializeInvalidArrayValue .ShouldThrow() .Message @@ -585,7 +623,7 @@ public void ShouldFailToDeserializeJsonMapWithUnexpectedFormat() //Metadata declares that "arrayProperty" whose items should be { objects }, not booleans. Action attemptToDeserializeInvalidArrayItemValue = - () => Deserialize(_resourceMetadata, @"{ ""arrayProperty"": [ true, false ] }"); + () => DeserializeNormalMap(_resourceMetadata, @"{ ""arrayProperty"": [ true, false ] }"); attemptToDeserializeInvalidArrayItemValue .ShouldThrow() .Message @@ -595,7 +633,7 @@ public void ShouldFailToDeserializeJsonMapWithUnexpectedFormat() //Metadata declares that "propertyA" should be Column Source object like a column mapping or static value, not an array. Action attemptToDeserializeInvalidColumnMapValue = - () => Deserialize(_resourceMetadata, @"{ ""propertyA"": [] }"); + () => DeserializeNormalMap(_resourceMetadata, @"{ ""propertyA"": [] }"); attemptToDeserializeInvalidColumnMapValue .ShouldThrow() .Message @@ -605,7 +643,7 @@ public void ShouldFailToDeserializeJsonMapWithUnexpectedFormat() //When a Column Source object is expected, it should have a "Column" property. Action attemptToDeserializeMissingSourceColumn = - () => Deserialize(_resourceMetadata, @"{ ""propertyA"": {} }"); + () => DeserializeNormalMap(_resourceMetadata, @"{ ""propertyA"": {} }"); attemptToDeserializeMissingSourceColumn .ShouldThrow() .Message @@ -617,7 +655,7 @@ public void ShouldFailToDeserializeJsonMapWithUnexpectedFormat() //When a Column Source object is expected, it can have optional "Default" and "Lookup" properties //in addition to "Column", but no other unexpected properties. Action attemptToDeserializeUnexpectedColumnSourceProperties = - () => Deserialize(_resourceMetadata, @" + () => DeserializeNormalMap(_resourceMetadata, @" { ""propertyA"": { ""Column"": ""Col1"", @@ -636,7 +674,7 @@ public void ShouldFailToDeserializeJsonMapWithUnexpectedFormat() //When a Column Source object is expected, and has expected keys, Columns and Lookup must be strings. Action attemptToDeserializeNonStringColumnSourceProperties = - () => Deserialize(_resourceMetadata, @" + () => DeserializeNormalMap(_resourceMetadata, @" { ""propertyA"": { ""Column"": null, @@ -654,7 +692,7 @@ public void ShouldFailToDeserializeJsonMapWithUnexpectedFormat() //When a Column Source object is expected, and has expected keys, Default must be a single value. Action attemptToDeserializeComplexDefault = - () => Deserialize(_resourceMetadata, @" + () => DeserializeNormalMap(_resourceMetadata, @" { ""propertyA"": { ""Column"": ""Col1"", @@ -764,12 +802,12 @@ public void ShouldSerializeAndDeserializeAtypicalArrayMappings() ] }"; - Serialize(atypicalResourceMetadata, atypicalMappings).ShouldMatch(atypicalJsonMap); - Deserialize(atypicalResourceMetadata, atypicalJsonMap).ShouldMatch(atypicalMappings); + SerializeNormalMap(atypicalResourceMetadata, atypicalMappings).ShouldMatch(atypicalJsonMap); + DeserializeNormalMap(atypicalResourceMetadata, atypicalJsonMap).ShouldMatch(atypicalMappings); //Metadata declares that "booleanArrayProperty" array elements should be Column Source object like a column mapping or static value, not an array. Action attemptToDeserializeInvalidColumnMapValue = - () => Deserialize(atypicalResourceMetadata, @"{ + () => DeserializeNormalMap(atypicalResourceMetadata, @"{ ""booleanArrayProperty"": [ { ""Column"": ""ColumnK"" @@ -795,7 +833,7 @@ public void ShouldFailToSerializeAmbiguousMappings() //to serialize at all. Action attemptToSerializeAmbiguousMapping = - () => Serialize(_resourceMetadata, new[] + () => SerializeNormalMap(_resourceMetadata, new[] { new DataMapper { @@ -869,10 +907,10 @@ public void ShouldSerializeAndDeserializeStaticValuesRespectingDataTypeWhenLossl ""unanticipatedType"": ""some future unanticipated swagger type value"" }"; - Serialize(resourceMetadata, mappings) + SerializeNormalMap(resourceMetadata, mappings) .ShouldMatch(jsonMap); - Deserialize(resourceMetadata, jsonMap) + DeserializeNormalMap(resourceMetadata, jsonMap) .ShouldMatch( MapStatic("stringProperty", "ABC123"), //Trimmed. MapStatic("dateTimeProperty", "08-01-2017"), //Trimmed. @@ -976,10 +1014,10 @@ public void ShouldSerializeAndDeserializeDefaultValuesRespectingDataTypeWhenLoss } }"; - Serialize(resourceMetadata, mappings) + SerializeNormalMap(resourceMetadata, mappings) .ShouldMatch(jsonMap); - Deserialize(resourceMetadata, jsonMap) + DeserializeNormalMap(resourceMetadata, jsonMap) .ShouldMatch( MapColumn("stringProperty", "ColumnA", "ABC123"), //Trimmed. MapColumn("dateTimeProperty", "ColumnB", "08-01-2017"), //Trimmed. @@ -995,25 +1033,46 @@ public void ShouldSerializeAndDeserializeDefaultValuesRespectingDataTypeWhenLoss ); } - private static JToken Serialize(ResourceMetadata[] resourceMetadata, DataMapper[] mappings) + private static JToken SerializeNormalMap(ResourceMetadata[] resourceMetadata, DataMapper[] mappings) { var dataMapSerializer = new DataMapSerializer("/testResource", resourceMetadata); return JToken.Parse(dataMapSerializer.Serialize(mappings)); } - private static DataMapper[] Deserialize(ResourceMetadata[] resourceMetadata, string jsonMap) + private static JToken SerializeDeleteByIdMap(DataMapper[] mappings) + { + var dataMapDeleteSerializer = new DeleteDataMapSerializer(); + + return JToken.Parse(dataMapDeleteSerializer.Serialize(mappings)); + } + + private static DataMapper[] DeserializeNormalMap(ResourceMetadata[] resourceMetadata, string jsonMap) { var dataMapSerializer = new DataMapSerializer("/testResource", resourceMetadata); return dataMapSerializer.Deserialize(jsonMap); } - private static DataMapper[] Deserialize(ResourceMetadata[] resourceMetadata, JObject jsonMap) + private static DataMapper[] DeserializeDeleteByIdMap(ResourceMetadata[] resourceMetadata, string jsonMap) + { + var dataMapSerializer = new DeleteDataMapSerializer(); + + return dataMapSerializer.Deserialize(jsonMap); + } + + private static DataMapper[] DeserializeNormalMap(ResourceMetadata[] resourceMetadata, JObject jsonMap) { var dataMapSerializer = new DataMapSerializer("/testResource", resourceMetadata); return dataMapSerializer.Deserialize(jsonMap); } + + private static DataMapper[] DeserializeDeleteByIdMap(JObject jsonMap) + { + var dataMapSerializer = new DeleteDataMapSerializer(); + + return dataMapSerializer.Deserialize(jsonMap); + } } } diff --git a/DataImport.Models/DataMapSerializer.cs b/DataImport.Models/DataMapSerializer.cs index 2fe9673c..08dad43a 100644 --- a/DataImport.Models/DataMapSerializer.cs +++ b/DataImport.Models/DataMapSerializer.cs @@ -168,7 +168,7 @@ private List DeserializeObject(IReadOnlyList nodeM foreach (var node in nodes) { - if (nodeMetadatas.All(x => x.Name != node.Name)) + if (node.Name != "Id" && nodeMetadatas.All(x => x.Name != node.Name)) throw new InvalidOperationException( $"Cannot deserialize mappings from JSON, because the key '{node.Name}' should not exist " + $"according to the metadata for resource '{_resourcePath}'."); diff --git a/DataImport.Models/Datamap.cs b/DataImport.Models/Datamap.cs index 90ace03c..8606fbc1 100644 --- a/DataImport.Models/Datamap.cs +++ b/DataImport.Models/Datamap.cs @@ -48,5 +48,7 @@ public DataMap() public Script FileProcessorScript { get; set; } public ICollection DataMapAgents { get; set; } + + public bool IsDeleteOperation { get; set; } } } diff --git a/DataImport.Models/DeleteDataMapSerializer.cs b/DataImport.Models/DeleteDataMapSerializer.cs new file mode 100644 index 00000000..f2e106fe --- /dev/null +++ b/DataImport.Models/DeleteDataMapSerializer.cs @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace DataImport.Models +{ + public class DeleteDataMapSerializer + { + public string Serialize(DataMapper[] mappings) + { + return SerializeObjectForDeleteById(mappings.Single()).ToString(Formatting.Indented); + } + + public DataMapper[] Deserialize(string jsonMap) + { + JObject jobject; + try + { + jobject = JObject.Parse(jsonMap); + } + catch (Exception exception) + { + throw new ArgumentException( + "Cannot deserialize mappings from JSON, because the map text is not a valid JSON object. " + + "Check the inner exception for details. Invalid JSON Map text:" + + $"{Environment.NewLine}{Environment.NewLine}{jsonMap}" + , exception); + } + + return Deserialize(jobject); + } + public DataMapper[] Deserialize(JObject jsonMap) + { + return DeserializeObjectForDeleteById(jsonMap).ToArray(); + } + + private JObject SerializeObjectForDeleteById(DataMapper node) + { + var result = new JObject { new JProperty("Id", new JObject { new JProperty("Column", node.SourceColumn) }) }; + return result; + } + + private List DeserializeObjectForDeleteById(JToken objectToken) + { + var jobject = objectToken as JObject; + + if (jobject == null) + throw new InvalidOperationException( + "Cannot deserialize mappings from JSON, because an object literal was expected. " + + "Instead, found: " + + $"{objectToken.ToString(Formatting.Indented)}"); + + var result = new List(); + + var nodes = jobject.Children().Cast().ToArray(); + + var node = nodes.Single(n => n.Name == "Id"); + + var propertyValue = node.Children().Single(); + + var sourceColumn = ((JObject) propertyValue).Children().Cast().Single().Value; + + result.Add(new DataMapper() { Name = "Id", SourceColumn = DeserializeRawValue(sourceColumn) }); + + return result; + } + + private static string DeserializeRawValue(object rawValue) + { + if (rawValue == null) + return null; + + if (rawValue is bool) + return rawValue.ToString().ToLower(); + + return rawValue.ToString(); + } + } +} diff --git a/DataImport.Models/Migrations/PostgreSql/20231128143229_AddIsDeleteOperationField.Designer.cs b/DataImport.Models/Migrations/PostgreSql/20231128143229_AddIsDeleteOperationField.Designer.cs new file mode 100644 index 00000000..e3761c4d --- /dev/null +++ b/DataImport.Models/Migrations/PostgreSql/20231128143229_AddIsDeleteOperationField.Designer.cs @@ -0,0 +1,1123 @@ +// +using System; +using DataImport.Models; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace DataImport.Models.Migrations.PostgreSql +{ + [DbContext(typeof(PostgreSqlDataImportDbContext))] + [Migration("20231128143229_AddIsDeleteOperationField")] + partial class AddIsDeleteOperationField + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "6.0.1") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("DataImport.Models.AdminView", b => + { + b.Property("Email") + .HasColumnType("text"); + + b.ToView("AdminView"); + }); + + modelBuilder.Entity("DataImport.Models.Agent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AgentAction") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("AgentTypeCode") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("ApiServerId") + .HasColumnType("integer"); + + b.Property("Archived") + .HasColumnType("boolean"); + + b.Property("Created") + .HasColumnType("timestamp with time zone"); + + b.Property("Directory") + .HasColumnType("text"); + + b.Property("Enabled") + .HasColumnType("boolean"); + + b.Property("FileGeneratorScriptId") + .HasColumnType("integer"); + + b.Property("FilePattern") + .HasColumnType("text"); + + b.Property("LastExecuted") + .HasColumnType("timestamp with time zone"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("Password") + .HasColumnType("text"); + + b.Property("Port") + .HasColumnType("integer"); + + b.Property("RowProcessorScriptId") + .HasColumnType("integer"); + + b.Property("RunOrder") + .HasColumnType("integer"); + + b.Property("Url") + .HasColumnType("text"); + + b.Property("Username") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("ApiServerId"); + + b.HasIndex("FileGeneratorScriptId"); + + b.HasIndex("RowProcessorScriptId"); + + b.ToTable("Agents"); + }); + + modelBuilder.Entity("DataImport.Models.AgentSchedule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AgentId") + .HasColumnType("integer"); + + b.Property("Day") + .HasColumnType("integer"); + + b.Property("Hour") + .HasColumnType("integer"); + + b.Property("Minute") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("AgentId"); + + b.ToTable("AgentSchedules"); + }); + + modelBuilder.Entity("DataImport.Models.ApiServer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ApiVersionId") + .HasColumnType("integer"); + + b.Property("AuthUrl") + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("Key") + .IsRequired() + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("Secret") + .IsRequired() + .HasColumnType("text"); + + b.Property("TokenUrl") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("Url") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.HasKey("Id"); + + b.HasIndex("ApiVersionId"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("ApiServers"); + }); + + modelBuilder.Entity("DataImport.Models.ApiVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Version") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)") + .HasColumnName("ApiVersion"); + + b.HasKey("Id"); + + b.HasIndex("Version") + .IsUnique(); + + b.ToTable("ApiVersions"); + }); + + modelBuilder.Entity("DataImport.Models.ApplicationLog", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Exception") + .HasColumnType("text"); + + b.Property("Level") + .IsRequired() + .HasMaxLength(5) + .HasColumnType("character varying(5)"); + + b.Property("Logged") + .HasColumnType("timestamp with time zone"); + + b.Property("Logger") + .HasMaxLength(300) + .HasColumnType("character varying(300)"); + + b.Property("MachineName") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Message") + .HasColumnType("text"); + + b.Property("Port") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Properties") + .HasColumnType("text"); + + b.Property("RemoteAddress") + .HasColumnType("text"); + + b.Property("ServerAddress") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("ServerName") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Url") + .HasMaxLength(2000) + .HasColumnType("character varying(2000)"); + + b.Property("UserName") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.HasKey("Id"); + + b.HasIndex("Logged"); + + b.ToTable("ApplicationLogs"); + }); + + modelBuilder.Entity("DataImport.Models.BootstrapData", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ApiVersionId") + .HasColumnType("integer"); + + b.Property("CreateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Data") + .IsRequired() + .HasColumnType("text"); + + b.Property("Metadata") + .IsRequired() + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("ResourcePath") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("UpdateDate") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("ApiVersionId"); + + b.ToTable("BootstrapDatas"); + }); + + modelBuilder.Entity("DataImport.Models.BootstrapDataAgent", b => + { + b.Property("BootstrapDataId") + .HasColumnType("integer"); + + b.Property("AgentId") + .HasColumnType("integer"); + + b.Property("ProcessingOrder") + .HasColumnType("integer"); + + b.HasKey("BootstrapDataId", "AgentId"); + + b.HasIndex("AgentId"); + + b.ToTable("BootstrapDataAgents"); + }); + + modelBuilder.Entity("DataImport.Models.BootstrapDataApiServer", b => + { + b.Property("BootstrapDataId") + .HasColumnType("integer"); + + b.Property("ApiServerId") + .HasColumnType("integer"); + + b.Property("ProcessedDate") + .HasColumnType("timestamp with time zone"); + + b.HasKey("BootstrapDataId", "ApiServerId"); + + b.HasIndex("ApiServerId"); + + b.ToTable("BootstrapDataApiServers"); + }); + + modelBuilder.Entity("DataImport.Models.Configuration", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AvailableCmdlets") + .HasColumnType("text"); + + b.Property("ImportPSModules") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Configurations"); + }); + + modelBuilder.Entity("DataImport.Models.DatabaseVersion", b => + { + b.Property("VersionString") + .HasColumnType("text"); + + b.ToTable("DatabaseVersion"); + + b.ToSqlQuery("SELECT version() as VersionString"); + }); + + modelBuilder.Entity("DataImport.Models.DataMap", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ApiVersionId") + .HasColumnType("integer"); + + b.Property("Attribute") + .HasColumnType("text"); + + b.Property("ColumnHeaders") + .HasColumnType("text"); + + b.Property("CreateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("FileProcessorScriptId") + .HasColumnType("integer"); + + b.Property("IsDeleteOperation") + .HasColumnType("boolean"); + + b.Property("Map") + .IsRequired() + .HasColumnType("text"); + + b.Property("Metadata") + .IsRequired() + .HasColumnType("text"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("ResourcePath") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("UpdateDate") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("ApiVersionId"); + + b.HasIndex("FileProcessorScriptId"); + + b.ToTable("DataMaps"); + }); + + modelBuilder.Entity("DataImport.Models.DataMapAgent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AgentId") + .HasColumnType("integer"); + + b.Property("DataMapId") + .HasColumnType("integer"); + + b.Property("ProcessingOrder") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("AgentId"); + + b.HasIndex("DataMapId"); + + b.ToTable("DataMapAgents"); + }); + + modelBuilder.Entity("DataImport.Models.File", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AgentId") + .HasColumnType("integer"); + + b.Property("CreateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("FileName") + .IsRequired() + .HasColumnType("text"); + + b.Property("Message") + .HasColumnType("text"); + + b.Property("Rows") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("integer"); + + b.Property("UpdateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Url") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("AgentId"); + + b.HasIndex("CreateDate"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("DataImport.Models.Identity.ApplicationUser", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("AccessFailedCount") + .HasColumnType("integer"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("text"); + + b.Property("Email") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("EmailConfirmed") + .HasColumnType("boolean"); + + b.Property("LockoutEnabled") + .HasColumnType("boolean"); + + b.Property("LockoutEnd") + .HasColumnType("timestamp with time zone"); + + b.Property("NormalizedEmail") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("NormalizedUserName") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("PasswordHash") + .HasColumnType("text"); + + b.Property("PhoneNumber") + .HasColumnType("text"); + + b.Property("PhoneNumberConfirmed") + .HasColumnType("boolean"); + + b.Property("SecurityStamp") + .HasColumnType("text"); + + b.Property("TwoFactorEnabled") + .HasColumnType("boolean"); + + b.Property("UserName") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedEmail") + .HasDatabaseName("EmailIndex"); + + b.HasIndex("NormalizedUserName") + .IsUnique() + .HasDatabaseName("UserNameIndex"); + + b.ToTable("AspNetUsers", (string)null); + }); + + modelBuilder.Entity("DataImport.Models.IngestionLog", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AgentName") + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("ApiServerName") + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("ApiVersion") + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("Data") + .HasColumnType("text"); + + b.Property("Date") + .HasColumnType("timestamp with time zone"); + + b.Property("EducationOrganizationId") + .HasColumnType("uuid"); + + b.Property("EndPointUrl") + .HasColumnType("text"); + + b.Property("FileName") + .HasColumnType("text"); + + b.Property("HttpStatusCode") + .HasColumnType("text"); + + b.Property("Level") + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("OdsResponse") + .HasColumnType("text"); + + b.Property("Operation") + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("Process") + .HasColumnType("text"); + + b.Property("Result") + .HasColumnType("integer"); + + b.Property("RowNumber") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("Date"); + + b.HasIndex("Result"); + + b.ToTable("IngestionLogs"); + }); + + modelBuilder.Entity("DataImport.Models.JobStatus", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Completed") + .HasColumnType("timestamp with time zone"); + + b.Property("Started") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("JobStatus"); + }); + + modelBuilder.Entity("DataImport.Models.Lookup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Key") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("SourceTable") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.Property("Value") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("character varying(1024)"); + + b.HasKey("Id"); + + b.HasIndex("SourceTable", "Key") + .IsUnique(); + + b.ToTable("Lookups"); + }); + + modelBuilder.Entity("DataImport.Models.Resource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ApiSection") + .HasColumnType("integer"); + + b.Property("ApiVersionId") + .HasColumnType("integer"); + + b.Property("Metadata") + .IsRequired() + .HasColumnType("text"); + + b.Property("Path") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.HasKey("Id"); + + b.HasIndex("ApiVersionId"); + + b.HasIndex("Path", "ApiVersionId") + .IsUnique(); + + b.ToTable("Resources"); + }); + + modelBuilder.Entity("DataImport.Models.Script", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ExecutableArguments") + .HasColumnType("text"); + + b.Property("ExecutablePath") + .HasColumnType("text"); + + b.Property("HasAttribute") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + + b.Property("RequireOdsApiAccess") + .HasColumnType("boolean"); + + b.Property("ScriptContent") + .HasColumnType("text"); + + b.Property("ScriptType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.HasKey("Id"); + + b.HasIndex("Name", "ScriptType") + .IsUnique(); + + b.ToTable("Scripts"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole", b => + { + b.Property("Id") + .HasColumnType("text"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("text"); + + b.Property("Name") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.Property("NormalizedName") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedName") + .IsUnique() + .HasDatabaseName("RoleNameIndex"); + + b.ToTable("AspNetRoles", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ClaimType") + .HasColumnType("text"); + + b.Property("ClaimValue") + .HasColumnType("text"); + + b.Property("RoleId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetRoleClaims", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ClaimType") + .HasColumnType("text"); + + b.Property("ClaimValue") + .HasColumnType("text"); + + b.Property("UserId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserClaims", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.Property("LoginProvider") + .HasColumnType("text"); + + b.Property("ProviderKey") + .HasColumnType("text"); + + b.Property("ProviderDisplayName") + .HasColumnType("text"); + + b.Property("UserId") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("LoginProvider", "ProviderKey"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserLogins", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.Property("UserId") + .HasColumnType("text"); + + b.Property("RoleId") + .HasColumnType("text"); + + b.HasKey("UserId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetUserRoles", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.Property("UserId") + .HasColumnType("text"); + + b.Property("LoginProvider") + .HasColumnType("text"); + + b.Property("Name") + .HasColumnType("text"); + + b.Property("Value") + .HasColumnType("text"); + + b.HasKey("UserId", "LoginProvider", "Name"); + + b.ToTable("AspNetUserTokens", (string)null); + }); + + modelBuilder.Entity("DataImport.Models.Agent", b => + { + b.HasOne("DataImport.Models.ApiServer", "ApiServer") + .WithMany() + .HasForeignKey("ApiServerId"); + + b.HasOne("DataImport.Models.Script", "FileGenerator") + .WithMany() + .HasForeignKey("FileGeneratorScriptId"); + + b.HasOne("DataImport.Models.Script", "RowProcessor") + .WithMany() + .HasForeignKey("RowProcessorScriptId"); + + b.Navigation("ApiServer"); + + b.Navigation("FileGenerator"); + + b.Navigation("RowProcessor"); + }); + + modelBuilder.Entity("DataImport.Models.AgentSchedule", b => + { + b.HasOne("DataImport.Models.Agent", "Agent") + .WithMany("AgentSchedules") + .HasForeignKey("AgentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Agent"); + }); + + modelBuilder.Entity("DataImport.Models.ApiServer", b => + { + b.HasOne("DataImport.Models.ApiVersion", "ApiVersion") + .WithMany() + .HasForeignKey("ApiVersionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("ApiVersion"); + }); + + modelBuilder.Entity("DataImport.Models.BootstrapData", b => + { + b.HasOne("DataImport.Models.ApiVersion", "ApiVersion") + .WithMany() + .HasForeignKey("ApiVersionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("ApiVersion"); + }); + + modelBuilder.Entity("DataImport.Models.BootstrapDataAgent", b => + { + b.HasOne("DataImport.Models.Agent", "Agent") + .WithMany("BootstrapDataAgents") + .HasForeignKey("AgentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("DataImport.Models.BootstrapData", "BootstrapData") + .WithMany("BootstrapDataAgents") + .HasForeignKey("BootstrapDataId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Agent"); + + b.Navigation("BootstrapData"); + }); + + modelBuilder.Entity("DataImport.Models.BootstrapDataApiServer", b => + { + b.HasOne("DataImport.Models.ApiServer", "ApiServer") + .WithMany("BootstrapDataApiServers") + .HasForeignKey("ApiServerId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("DataImport.Models.BootstrapData", "BootstrapData") + .WithMany("BootstrapDataApiServers") + .HasForeignKey("BootstrapDataId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("ApiServer"); + + b.Navigation("BootstrapData"); + }); + + modelBuilder.Entity("DataImport.Models.DataMap", b => + { + b.HasOne("DataImport.Models.ApiVersion", "ApiVersion") + .WithMany() + .HasForeignKey("ApiVersionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("DataImport.Models.Script", "FileProcessorScript") + .WithMany("DataMaps") + .HasForeignKey("FileProcessorScriptId"); + + b.Navigation("ApiVersion"); + + b.Navigation("FileProcessorScript"); + }); + + modelBuilder.Entity("DataImport.Models.DataMapAgent", b => + { + b.HasOne("DataImport.Models.Agent", "Agent") + .WithMany("DataMapAgents") + .HasForeignKey("AgentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("DataImport.Models.DataMap", "DataMap") + .WithMany("DataMapAgents") + .HasForeignKey("DataMapId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Agent"); + + b.Navigation("DataMap"); + }); + + modelBuilder.Entity("DataImport.Models.File", b => + { + b.HasOne("DataImport.Models.Agent", "Agent") + .WithMany("Files") + .HasForeignKey("AgentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Agent"); + }); + + modelBuilder.Entity("DataImport.Models.Resource", b => + { + b.HasOne("DataImport.Models.ApiVersion", "ApiVersion") + .WithMany() + .HasForeignKey("ApiVersionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("ApiVersion"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.HasOne("DataImport.Models.Identity.ApplicationUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.HasOne("DataImport.Models.Identity.ApplicationUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DataImport.Models.Identity.ApplicationUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.HasOne("DataImport.Models.Identity.ApplicationUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DataImport.Models.Agent", b => + { + b.Navigation("AgentSchedules"); + + b.Navigation("BootstrapDataAgents"); + + b.Navigation("DataMapAgents"); + + b.Navigation("Files"); + }); + + modelBuilder.Entity("DataImport.Models.ApiServer", b => + { + b.Navigation("BootstrapDataApiServers"); + }); + + modelBuilder.Entity("DataImport.Models.BootstrapData", b => + { + b.Navigation("BootstrapDataAgents"); + + b.Navigation("BootstrapDataApiServers"); + }); + + modelBuilder.Entity("DataImport.Models.DataMap", b => + { + b.Navigation("DataMapAgents"); + }); + + modelBuilder.Entity("DataImport.Models.Script", b => + { + b.Navigation("DataMaps"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DataImport.Models/Migrations/PostgreSql/20231128143229_AddIsDeleteOperationField.cs b/DataImport.Models/Migrations/PostgreSql/20231128143229_AddIsDeleteOperationField.cs new file mode 100644 index 00000000..c412a627 --- /dev/null +++ b/DataImport.Models/Migrations/PostgreSql/20231128143229_AddIsDeleteOperationField.cs @@ -0,0 +1,26 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DataImport.Models.Migrations.PostgreSql +{ + public partial class AddIsDeleteOperationField : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "IsDeleteOperation", + table: "DataMaps", + type: "boolean", + nullable: false, + defaultValue: false); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "IsDeleteOperation", + table: "DataMaps"); + } + } +} diff --git a/DataImport.Models/Migrations/PostgreSql/PostgreSqlDataImportDbContextModelSnapshot.cs b/DataImport.Models/Migrations/PostgreSql/PostgreSqlDataImportDbContextModelSnapshot.cs index 764b532c..6b78e5bc 100644 --- a/DataImport.Models/Migrations/PostgreSql/PostgreSqlDataImportDbContextModelSnapshot.cs +++ b/DataImport.Models/Migrations/PostgreSql/PostgreSqlDataImportDbContextModelSnapshot.cs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: Apache-2.0 // Licensed to the Ed-Fi Alliance under one or more agreements. // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. @@ -399,6 +399,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("FileProcessorScriptId") .HasColumnType("integer"); + b.Property("IsDeleteOperation") + .HasColumnType("boolean"); + b.Property("Map") .IsRequired() .HasColumnType("text"); diff --git a/DataImport.Models/Migrations/SqlServer/20231127222509_AddIsDeleteOperationField.Designer.cs b/DataImport.Models/Migrations/SqlServer/20231127222509_AddIsDeleteOperationField.Designer.cs new file mode 100644 index 00000000..1a845053 --- /dev/null +++ b/DataImport.Models/Migrations/SqlServer/20231127222509_AddIsDeleteOperationField.Designer.cs @@ -0,0 +1,1125 @@ +// +using System; +using DataImport.Models; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace DataImport.Models.Migrations.SqlServer +{ + [DbContext(typeof(SqlDataImportDbContext))] + [Migration("20231127222509_AddIsDeleteOperationField")] + partial class AddIsDeleteOperationField + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "6.0.1") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder, 1L, 1); + + modelBuilder.Entity("DataImport.Models.AdminView", b => + { + b.Property("Email") + .HasColumnType("nvarchar(max)"); + + b.ToView("AdminView"); + }); + + modelBuilder.Entity("DataImport.Models.Agent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("AgentAction") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("AgentTypeCode") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("ApiServerId") + .HasColumnType("int"); + + b.Property("Archived") + .HasColumnType("bit"); + + b.Property("Created") + .HasColumnType("datetimeoffset"); + + b.Property("Directory") + .HasColumnType("nvarchar(max)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("FileGeneratorScriptId") + .HasColumnType("int"); + + b.Property("FilePattern") + .HasColumnType("nvarchar(max)"); + + b.Property("LastExecuted") + .HasColumnType("datetimeoffset"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("Password") + .HasColumnType("nvarchar(max)"); + + b.Property("Port") + .HasColumnType("int"); + + b.Property("RowProcessorScriptId") + .HasColumnType("int"); + + b.Property("RunOrder") + .HasColumnType("int"); + + b.Property("Url") + .HasColumnType("nvarchar(max)"); + + b.Property("Username") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("ApiServerId"); + + b.HasIndex("FileGeneratorScriptId"); + + b.HasIndex("RowProcessorScriptId"); + + b.ToTable("Agents"); + }); + + modelBuilder.Entity("DataImport.Models.AgentSchedule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("AgentId") + .HasColumnType("int"); + + b.Property("Day") + .HasColumnType("int"); + + b.Property("Hour") + .HasColumnType("int"); + + b.Property("Minute") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("AgentId"); + + b.ToTable("AgentSchedules"); + }); + + modelBuilder.Entity("DataImport.Models.ApiServer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("ApiVersionId") + .HasColumnType("int"); + + b.Property("AuthUrl") + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("Key") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("Secret") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("TokenUrl") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("Url") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.HasKey("Id"); + + b.HasIndex("ApiVersionId"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("ApiServers"); + }); + + modelBuilder.Entity("DataImport.Models.ApiVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("Version") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("nvarchar(20)") + .HasColumnName("ApiVersion"); + + b.HasKey("Id"); + + b.HasIndex("Version") + .IsUnique(); + + b.ToTable("ApiVersions"); + }); + + modelBuilder.Entity("DataImport.Models.ApplicationLog", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("Exception") + .HasColumnType("nvarchar(max)"); + + b.Property("Level") + .IsRequired() + .HasMaxLength(5) + .HasColumnType("nvarchar(5)"); + + b.Property("Logged") + .HasColumnType("datetimeoffset"); + + b.Property("Logger") + .HasMaxLength(300) + .HasColumnType("nvarchar(300)"); + + b.Property("MachineName") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Message") + .HasColumnType("nvarchar(max)"); + + b.Property("Port") + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("Properties") + .HasColumnType("nvarchar(max)"); + + b.Property("RemoteAddress") + .HasColumnType("nvarchar(max)"); + + b.Property("ServerAddress") + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("ServerName") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Url") + .HasMaxLength(2000) + .HasColumnType("nvarchar(2000)"); + + b.Property("UserName") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id"); + + b.HasIndex("Logged"); + + b.ToTable("ApplicationLogs"); + }); + + modelBuilder.Entity("DataImport.Models.BootstrapData", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("ApiVersionId") + .HasColumnType("int"); + + b.Property("CreateDate") + .HasColumnType("datetimeoffset"); + + b.Property("Data") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Metadata") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("ResourcePath") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("UpdateDate") + .HasColumnType("datetimeoffset"); + + b.HasKey("Id"); + + b.HasIndex("ApiVersionId"); + + b.ToTable("BootstrapDatas"); + }); + + modelBuilder.Entity("DataImport.Models.BootstrapDataAgent", b => + { + b.Property("BootstrapDataId") + .HasColumnType("int"); + + b.Property("AgentId") + .HasColumnType("int"); + + b.Property("ProcessingOrder") + .HasColumnType("int"); + + b.HasKey("BootstrapDataId", "AgentId"); + + b.HasIndex("AgentId"); + + b.ToTable("BootstrapDataAgents"); + }); + + modelBuilder.Entity("DataImport.Models.BootstrapDataApiServer", b => + { + b.Property("BootstrapDataId") + .HasColumnType("int"); + + b.Property("ApiServerId") + .HasColumnType("int"); + + b.Property("ProcessedDate") + .HasColumnType("datetimeoffset"); + + b.HasKey("BootstrapDataId", "ApiServerId"); + + b.HasIndex("ApiServerId"); + + b.ToTable("BootstrapDataApiServers"); + }); + + modelBuilder.Entity("DataImport.Models.Configuration", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("AvailableCmdlets") + .HasColumnType("nvarchar(max)"); + + b.Property("ImportPSModules") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("Configurations"); + }); + + modelBuilder.Entity("DataImport.Models.DatabaseVersion", b => + { + b.Property("VersionString") + .HasColumnType("nvarchar(max)"); + + b.ToTable("DatabaseVersion"); + + b.ToSqlQuery("SELECT @@VERSION as VersionString"); + }); + + modelBuilder.Entity("DataImport.Models.DataMap", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("ApiVersionId") + .HasColumnType("int"); + + b.Property("Attribute") + .HasColumnType("nvarchar(max)"); + + b.Property("ColumnHeaders") + .HasColumnType("nvarchar(max)"); + + b.Property("CreateDate") + .HasColumnType("datetimeoffset"); + + b.Property("FileProcessorScriptId") + .HasColumnType("int"); + + b.Property("IsDeleteOperation") + .HasColumnType("bit"); + + b.Property("Map") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Metadata") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("ResourcePath") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("UpdateDate") + .HasColumnType("datetimeoffset"); + + b.HasKey("Id"); + + b.HasIndex("ApiVersionId"); + + b.HasIndex("FileProcessorScriptId"); + + b.ToTable("DataMaps"); + }); + + modelBuilder.Entity("DataImport.Models.DataMapAgent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("AgentId") + .HasColumnType("int"); + + b.Property("DataMapId") + .HasColumnType("int"); + + b.Property("ProcessingOrder") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("AgentId"); + + b.HasIndex("DataMapId"); + + b.ToTable("DataMapAgents"); + }); + + modelBuilder.Entity("DataImport.Models.File", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("AgentId") + .HasColumnType("int"); + + b.Property("CreateDate") + .HasColumnType("datetimeoffset"); + + b.Property("FileName") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Message") + .HasColumnType("nvarchar(max)"); + + b.Property("Rows") + .HasColumnType("int"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("UpdateDate") + .HasColumnType("datetimeoffset"); + + b.Property("Url") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("AgentId"); + + b.HasIndex("CreateDate"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("DataImport.Models.Identity.ApplicationUser", b => + { + b.Property("Id") + .HasColumnType("nvarchar(450)"); + + b.Property("AccessFailedCount") + .HasColumnType("int"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("nvarchar(max)"); + + b.Property("Email") + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("EmailConfirmed") + .HasColumnType("bit"); + + b.Property("LockoutEnabled") + .HasColumnType("bit"); + + b.Property("LockoutEnd") + .HasColumnType("datetimeoffset"); + + b.Property("NormalizedEmail") + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("NormalizedUserName") + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("PasswordHash") + .HasColumnType("nvarchar(max)"); + + b.Property("PhoneNumber") + .HasColumnType("nvarchar(max)"); + + b.Property("PhoneNumberConfirmed") + .HasColumnType("bit"); + + b.Property("SecurityStamp") + .HasColumnType("nvarchar(max)"); + + b.Property("TwoFactorEnabled") + .HasColumnType("bit"); + + b.Property("UserName") + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedEmail") + .HasDatabaseName("EmailIndex"); + + b.HasIndex("NormalizedUserName") + .IsUnique() + .HasDatabaseName("UserNameIndex") + .HasFilter("[NormalizedUserName] IS NOT NULL"); + + b.ToTable("AspNetUsers", (string)null); + }); + + modelBuilder.Entity("DataImport.Models.IngestionLog", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("AgentName") + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("ApiServerName") + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("ApiVersion") + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + + b.Property("Data") + .HasColumnType("nvarchar(max)"); + + b.Property("Date") + .HasColumnType("datetimeoffset"); + + b.Property("EducationOrganizationId") + .HasColumnType("uniqueidentifier"); + + b.Property("EndPointUrl") + .HasColumnType("nvarchar(max)"); + + b.Property("FileName") + .HasColumnType("nvarchar(max)"); + + b.Property("HttpStatusCode") + .HasColumnType("nvarchar(max)"); + + b.Property("Level") + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("OdsResponse") + .HasColumnType("nvarchar(max)"); + + b.Property("Operation") + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("Process") + .HasColumnType("nvarchar(max)"); + + b.Property("Result") + .HasColumnType("int"); + + b.Property("RowNumber") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("Date"); + + b.HasIndex("Result"); + + b.ToTable("IngestionLogs"); + }); + + modelBuilder.Entity("DataImport.Models.JobStatus", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("Completed") + .HasColumnType("datetimeoffset"); + + b.Property("Started") + .HasColumnType("datetimeoffset"); + + b.HasKey("Id"); + + b.ToTable("JobStatus"); + }); + + modelBuilder.Entity("DataImport.Models.Lookup", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("Key") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("SourceTable") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("Value") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.HasKey("Id"); + + b.HasIndex("SourceTable", "Key") + .IsUnique(); + + b.ToTable("Lookups"); + }); + + modelBuilder.Entity("DataImport.Models.Resource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("ApiSection") + .HasColumnType("int"); + + b.Property("ApiVersionId") + .HasColumnType("int"); + + b.Property("Metadata") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Path") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.HasKey("Id"); + + b.HasIndex("ApiVersionId"); + + b.HasIndex("Path", "ApiVersionId") + .IsUnique(); + + b.ToTable("Resources"); + }); + + modelBuilder.Entity("DataImport.Models.Script", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("ExecutableArguments") + .HasColumnType("nvarchar(max)"); + + b.Property("ExecutablePath") + .HasColumnType("nvarchar(max)"); + + b.Property("HasAttribute") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("RequireOdsApiAccess") + .HasColumnType("bit"); + + b.Property("ScriptContent") + .HasColumnType("nvarchar(max)"); + + b.Property("ScriptType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.HasKey("Id"); + + b.HasIndex("Name", "ScriptType") + .IsUnique(); + + b.ToTable("Scripts"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRole", b => + { + b.Property("Id") + .HasColumnType("nvarchar(450)"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("NormalizedName") + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedName") + .IsUnique() + .HasDatabaseName("RoleNameIndex") + .HasFilter("[NormalizedName] IS NOT NULL"); + + b.ToTable("AspNetRoles", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("ClaimType") + .HasColumnType("nvarchar(max)"); + + b.Property("ClaimValue") + .HasColumnType("nvarchar(max)"); + + b.Property("RoleId") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetRoleClaims", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id"), 1L, 1); + + b.Property("ClaimType") + .HasColumnType("nvarchar(max)"); + + b.Property("ClaimValue") + .HasColumnType("nvarchar(max)"); + + b.Property("UserId") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserClaims", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.Property("LoginProvider") + .HasColumnType("nvarchar(450)"); + + b.Property("ProviderKey") + .HasColumnType("nvarchar(450)"); + + b.Property("ProviderDisplayName") + .HasColumnType("nvarchar(max)"); + + b.Property("UserId") + .IsRequired() + .HasColumnType("nvarchar(450)"); + + b.HasKey("LoginProvider", "ProviderKey"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserLogins", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.Property("UserId") + .HasColumnType("nvarchar(450)"); + + b.Property("RoleId") + .HasColumnType("nvarchar(450)"); + + b.HasKey("UserId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetUserRoles", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.Property("UserId") + .HasColumnType("nvarchar(450)"); + + b.Property("LoginProvider") + .HasColumnType("nvarchar(450)"); + + b.Property("Name") + .HasColumnType("nvarchar(450)"); + + b.Property("Value") + .HasColumnType("nvarchar(max)"); + + b.HasKey("UserId", "LoginProvider", "Name"); + + b.ToTable("AspNetUserTokens", (string)null); + }); + + modelBuilder.Entity("DataImport.Models.Agent", b => + { + b.HasOne("DataImport.Models.ApiServer", "ApiServer") + .WithMany() + .HasForeignKey("ApiServerId"); + + b.HasOne("DataImport.Models.Script", "FileGenerator") + .WithMany() + .HasForeignKey("FileGeneratorScriptId"); + + b.HasOne("DataImport.Models.Script", "RowProcessor") + .WithMany() + .HasForeignKey("RowProcessorScriptId"); + + b.Navigation("ApiServer"); + + b.Navigation("FileGenerator"); + + b.Navigation("RowProcessor"); + }); + + modelBuilder.Entity("DataImport.Models.AgentSchedule", b => + { + b.HasOne("DataImport.Models.Agent", "Agent") + .WithMany("AgentSchedules") + .HasForeignKey("AgentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Agent"); + }); + + modelBuilder.Entity("DataImport.Models.ApiServer", b => + { + b.HasOne("DataImport.Models.ApiVersion", "ApiVersion") + .WithMany() + .HasForeignKey("ApiVersionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("ApiVersion"); + }); + + modelBuilder.Entity("DataImport.Models.BootstrapData", b => + { + b.HasOne("DataImport.Models.ApiVersion", "ApiVersion") + .WithMany() + .HasForeignKey("ApiVersionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("ApiVersion"); + }); + + modelBuilder.Entity("DataImport.Models.BootstrapDataAgent", b => + { + b.HasOne("DataImport.Models.Agent", "Agent") + .WithMany("BootstrapDataAgents") + .HasForeignKey("AgentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("DataImport.Models.BootstrapData", "BootstrapData") + .WithMany("BootstrapDataAgents") + .HasForeignKey("BootstrapDataId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Agent"); + + b.Navigation("BootstrapData"); + }); + + modelBuilder.Entity("DataImport.Models.BootstrapDataApiServer", b => + { + b.HasOne("DataImport.Models.ApiServer", "ApiServer") + .WithMany("BootstrapDataApiServers") + .HasForeignKey("ApiServerId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("DataImport.Models.BootstrapData", "BootstrapData") + .WithMany("BootstrapDataApiServers") + .HasForeignKey("BootstrapDataId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("ApiServer"); + + b.Navigation("BootstrapData"); + }); + + modelBuilder.Entity("DataImport.Models.DataMap", b => + { + b.HasOne("DataImport.Models.ApiVersion", "ApiVersion") + .WithMany() + .HasForeignKey("ApiVersionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("DataImport.Models.Script", "FileProcessorScript") + .WithMany("DataMaps") + .HasForeignKey("FileProcessorScriptId"); + + b.Navigation("ApiVersion"); + + b.Navigation("FileProcessorScript"); + }); + + modelBuilder.Entity("DataImport.Models.DataMapAgent", b => + { + b.HasOne("DataImport.Models.Agent", "Agent") + .WithMany("DataMapAgents") + .HasForeignKey("AgentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("DataImport.Models.DataMap", "DataMap") + .WithMany("DataMapAgents") + .HasForeignKey("DataMapId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Agent"); + + b.Navigation("DataMap"); + }); + + modelBuilder.Entity("DataImport.Models.File", b => + { + b.HasOne("DataImport.Models.Agent", "Agent") + .WithMany("Files") + .HasForeignKey("AgentId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Agent"); + }); + + modelBuilder.Entity("DataImport.Models.Resource", b => + { + b.HasOne("DataImport.Models.ApiVersion", "ApiVersion") + .WithMany() + .HasForeignKey("ApiVersionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("ApiVersion"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.HasOne("DataImport.Models.Identity.ApplicationUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.HasOne("DataImport.Models.Identity.ApplicationUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserRole", b => + { + b.HasOne("Microsoft.AspNetCore.Identity.IdentityRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DataImport.Models.Identity.ApplicationUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.HasOne("DataImport.Models.Identity.ApplicationUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DataImport.Models.Agent", b => + { + b.Navigation("AgentSchedules"); + + b.Navigation("BootstrapDataAgents"); + + b.Navigation("DataMapAgents"); + + b.Navigation("Files"); + }); + + modelBuilder.Entity("DataImport.Models.ApiServer", b => + { + b.Navigation("BootstrapDataApiServers"); + }); + + modelBuilder.Entity("DataImport.Models.BootstrapData", b => + { + b.Navigation("BootstrapDataAgents"); + + b.Navigation("BootstrapDataApiServers"); + }); + + modelBuilder.Entity("DataImport.Models.DataMap", b => + { + b.Navigation("DataMapAgents"); + }); + + modelBuilder.Entity("DataImport.Models.Script", b => + { + b.Navigation("DataMaps"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DataImport.Models/Migrations/SqlServer/20231127222509_AddIsDeleteOperationField.cs b/DataImport.Models/Migrations/SqlServer/20231127222509_AddIsDeleteOperationField.cs new file mode 100644 index 00000000..350c84a0 --- /dev/null +++ b/DataImport.Models/Migrations/SqlServer/20231127222509_AddIsDeleteOperationField.cs @@ -0,0 +1,26 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DataImport.Models.Migrations.SqlServer +{ + public partial class AddIsDeleteOperationField : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "IsDeleteOperation", + table: "DataMaps", + type: "bit", + nullable: false, + defaultValue: false); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "IsDeleteOperation", + table: "DataMaps"); + } + } +} diff --git a/DataImport.Models/Migrations/SqlServer/SqlDataImportDbContextModelSnapshot.cs b/DataImport.Models/Migrations/SqlServer/SqlDataImportDbContextModelSnapshot.cs index fc428ba1..02b30f80 100644 --- a/DataImport.Models/Migrations/SqlServer/SqlDataImportDbContextModelSnapshot.cs +++ b/DataImport.Models/Migrations/SqlServer/SqlDataImportDbContextModelSnapshot.cs @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: Apache-2.0 +// SPDX-License-Identifier: Apache-2.0 // Licensed to the Ed-Fi Alliance under one or more agreements. // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. @@ -399,6 +399,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("FileProcessorScriptId") .HasColumnType("int"); + b.Property("IsDeleteOperation") + .HasColumnType("bit"); + b.Property("Map") .IsRequired() .HasColumnType("nvarchar(max)"); diff --git a/DataImport.Server.TransformLoad.Tests/Features/LoadResources/TestFailingOdsApi.cs b/DataImport.Server.TransformLoad.Tests/Features/LoadResources/TestFailingOdsApi.cs index fb695966..fef41ac1 100644 --- a/DataImport.Server.TransformLoad.Tests/Features/LoadResources/TestFailingOdsApi.cs +++ b/DataImport.Server.TransformLoad.Tests/Features/LoadResources/TestFailingOdsApi.cs @@ -14,6 +14,11 @@ public class TestFailingOdsApi : IOdsApi { public static string ConfigUrlDefault = "http://test-ods-v2.5.0.1.example.com/api/v2.0/2019"; + public Task Delete(string id, string endpointUrl) + { + throw new NotImplementedException(); + } + public ApiConfig Config { get; set; } = new ApiConfig { ApiUrl = ConfigUrlDefault }; diff --git a/DataImport.Server.TransformLoad.Tests/Features/LoadResources/TestOdsApi.cs b/DataImport.Server.TransformLoad.Tests/Features/LoadResources/TestOdsApi.cs index db16e2bb..d5e57c44 100644 --- a/DataImport.Server.TransformLoad.Tests/Features/LoadResources/TestOdsApi.cs +++ b/DataImport.Server.TransformLoad.Tests/Features/LoadResources/TestOdsApi.cs @@ -16,12 +16,22 @@ public TestOdsApi() { PostedBootstrapData = new List(); PostedContent = new List(); + DeletedContent = new List(); } public List PostedContent { get; } + public List DeletedContent { get; } + public List PostedBootstrapData { get; } + public Task Delete(string id, string endpointUrl) + { + DeletedContent.Add(new SimulatedDelete(endpointUrl, id)); + + return Task.FromResult(new OdsResponse(HttpStatusCode.NoContent, string.Empty)); + } + public ApiConfig Config { get; set; } = new ApiConfig { ApiUrl = "http://test-ods-v2.5.0.1.example.com/api/v2.0/2019" }; @@ -50,5 +60,17 @@ public SimulatedPost(string endpointUrl, string body) public string EndpointUrl { get; set; } public string Body { get; set; } } + + public class SimulatedDelete + { + public SimulatedDelete(string endpointUrl, string id) + { + EndpointUrl = endpointUrl; + Id = id; + } + + public string EndpointUrl { get; set; } + public string Id { get; set; } + } } } diff --git a/DataImport.Server.TransformLoad/Features/LoadResources/FileProcessor.cs b/DataImport.Server.TransformLoad/Features/LoadResources/FileProcessor.cs index c04b4002..b5ba9034 100644 --- a/DataImport.Server.TransformLoad/Features/LoadResources/FileProcessor.cs +++ b/DataImport.Server.TransformLoad/Features/LoadResources/FileProcessor.cs @@ -21,6 +21,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Newtonsoft.Json.Linq; using static System.Environment; using File = DataImport.Models.File; using LogLevels = DataImport.Common.Enums.LogLevel; @@ -149,7 +150,7 @@ protected override async Task Handle(Command request, CancellationToken cancella UpdateStatus(file.Id, FileStatus.Transforming); - await TransformAndPostEachRowUsingDataMap(file, dataMap, request.OdsApi, agent); + await TransformAndProcessEachRowUsingDataMap(file, dataMap, request.OdsApi, agent); } } @@ -170,7 +171,7 @@ protected override async Task Handle(Command request, CancellationToken cancella } } - private async Task TransformAndPostEachRowUsingDataMap(File file, DataMap dataMap, IOdsApi odsApi, Agent agent) + private async Task TransformAndProcessEachRowUsingDataMap(File file, DataMap dataMap, IOdsApi odsApi, Agent agent) { string tempCsvFilePath = null; var successCount = 0; @@ -191,7 +192,7 @@ await Parallel.ForEachAsync(importRows, parallelOptions, async (row, token) => ValidateHeadersAndLookUps(file, dataMap, row.Content.Keys); _logger.LogDebug("Transforming {path} row {row}", dataMap.ResourcePath, row.Number); - var (rowPostResponse, log) = await MapAndPostCsvRow(odsApi, row.Content, dataMap, row.Number, file); + var (rowPostResponse, log) = await MapAndProcessCsvRow(odsApi, row.Content, dataMap, row.Number, file); if (log != null && _ingestionLogLevels.Contains(log.Level)) WriteLog(log); @@ -348,7 +349,7 @@ private void ThrowIfMetadataIsIncompatible(DataMap dataMap) } } - private Task<(RowResult, IngestionLogMarker)> MapAndPostCsvRow(IOdsApi odsApi, Dictionary currentRow, DataMap map, int rowNum, File file) + private Task<(RowResult, IngestionLogMarker)> MapAndProcessCsvRow(IOdsApi odsApi, Dictionary currentRow, DataMap map, int rowNum, File file) { MappedResource mappedRow = null; @@ -375,14 +376,18 @@ private void ThrowIfMetadataIsIncompatible(DataMap dataMap) return mappedRow == null ? Task.FromResult((RowResult.Error, (IngestionLogMarker) null)) - : PostMappedRow(odsApi, mappedRow, map.ResourcePath); + : map.IsDeleteOperation + ? DeleteMappedRow(odsApi, mappedRow, map.ResourcePath) + : PostMappedRow(odsApi, mappedRow, map.ResourcePath); } private MappedResource TransformCsvRow(DataMap dataMap, Dictionary currentRow, int rowNum, File file) { var resourceMapper = new ResourceMapper(_logger, dataMap, _mappingLookups); - var mappedRowJson = resourceMapper.ApplyMap(currentRow); + var mappedRowJson = dataMap.IsDeleteOperation + ? resourceMapper.ApplyMapForDeleteByIdOperation(currentRow) + : resourceMapper.ApplyMap(currentRow); return new MappedResource { @@ -400,9 +405,9 @@ private MappedResource TransformCsvRow(DataMap dataMap, Dictionary PostMappedRow(IOdsApi odsApi, MappedResource mappedRow, string resourcePath) { - var endpointUrl = $"{odsApi.Config.ApiUrl}{resourcePath}"; + var endpointUrl = $"{odsApi.Config.ApiUrl.TrimEnd('/')}{resourcePath}"; - if (RowHasAlreadyBeenPosted(mappedRow, endpointUrl)) + if (RowHasAlreadyBeenProcessed(mappedRow, endpointUrl)) return (RowResult.Duplicate, null); var postInfo = $"{mappedRow.ResourcePath} row {mappedRow.RowNumber}"; @@ -432,7 +437,39 @@ private MappedResource TransformCsvRow(DataMap dataMap, Dictionary DeleteMappedRow(IOdsApi odsApi, MappedResource mappedRow, string resourcePath) + { + var endpointUrl = $"{odsApi.Config.ApiUrl.TrimEnd('/')}{resourcePath}"; + + if (RowHasAlreadyBeenProcessed(mappedRow, endpointUrl)) + return (RowResult.Duplicate, null); + + _logger.LogDebug("Deleting {id}", mappedRow.Value.ToString()); + + OdsResponse odsResponse; + try + { + var id = mappedRow.Value.SelectToken("Id").Value(); + + odsResponse = await odsApi.Delete(id, endpointUrl); + } + catch (Exception ex) + { + _logger.LogError(ex, "POST failed for resource: {url}, Row Number: {row}", endpointUrl, mappedRow.RowNumber); + return (RowResult.Error, new IngestionLogMarker(IngestionResult.Error, LogLevels.Error, mappedRow, endpointUrl)); + } + + switch (odsResponse.StatusCode) + { + case HttpStatusCode.NoContent: + return (RowResult.Success, new IngestionLogMarker(IngestionResult.Success, LogLevels.Information, mappedRow, endpointUrl, odsResponse.StatusCode)); + default: + _logger.LogError($"DELETE returned unexpected HTTP status: {endpointUrl}, Row Number: {mappedRow.RowNumber}, Status: {odsResponse.StatusCode}, Error: {odsResponse.Content}"); + return (RowResult.Error, new IngestionLogMarker(IngestionResult.Error, LogLevels.Error, mappedRow, endpointUrl, odsResponse.StatusCode, odsResponse.Content)); + } + } + + private bool RowHasAlreadyBeenProcessed(MappedResource mappedResource, string endPoint) { var strBuilder = new StringBuilder(); strBuilder.AppendLine("Endpoint: " + endPoint); diff --git a/DataImport.Server.TransformLoad/Features/LoadResources/OdsApi.cs b/DataImport.Server.TransformLoad/Features/LoadResources/OdsApi.cs index cc0bc072..7fe80f88 100644 --- a/DataImport.Server.TransformLoad/Features/LoadResources/OdsApi.cs +++ b/DataImport.Server.TransformLoad/Features/LoadResources/OdsApi.cs @@ -20,6 +20,7 @@ public interface IOdsApi { Task PostBootstrapData(string endpointUrl, string dataToInsert); Task Post(string content, string endpointUrl, string postInfo = null); + Task Delete(string id, string endpointUrl); ApiConfig Config { get; set; } } @@ -171,5 +172,35 @@ public async Task Post(string content, string endpointUrl, string p return new OdsResponse(response.StatusCode, responseContent); } + + public async Task Delete(string id, string endpointUrl) + { + await Authenticate(); + + const int RetryAttempts = 3; + var currentAttempt = 0; + HttpResponseMessage response = null; + + while (RetryAttempts > currentAttempt) + { + response = await AuthenticatedHttpClient.Value.DeleteAsync($"{endpointUrl}/{id}"); + currentAttempt++; + + if (response.StatusCode == HttpStatusCode.Unauthorized) + { + AccessToken = null; + await Authenticate(); + AuthenticatedHttpClient = new Lazy(CreateAuthenticatedHttpClient); + _logger.LogWarning("DELETE failed. Reason: {reason}. StatusCode: {status}.", response.ReasonPhrase, response.StatusCode); + _logger.LogInformation("Refreshing token and retrying DELETE request for {id}.", id); + } + else + break; + } + + var responseContent = await response.Content.ReadAsStringAsync(); + + return new OdsResponse(response.StatusCode, responseContent); + } } } diff --git a/DataImport.Server.TransformLoad/Features/LoadResources/ResourceMapper.cs b/DataImport.Server.TransformLoad/Features/LoadResources/ResourceMapper.cs index 05c40a4e..642ec480 100644 --- a/DataImport.Server.TransformLoad/Features/LoadResources/ResourceMapper.cs +++ b/DataImport.Server.TransformLoad/Features/LoadResources/ResourceMapper.cs @@ -22,8 +22,9 @@ public ResourceMapper(ILogger logger, DataMap dataMap, LookupCollection mappingL { _logger = logger; - var dataMapSerializer = new DataMapSerializer(dataMap); - _mappings = dataMapSerializer.Deserialize(dataMap.Map); + _mappings = dataMap.IsDeleteOperation + ? new DeleteDataMapSerializer().Deserialize(dataMap.Map) + : new DataMapSerializer(dataMap).Deserialize(dataMap.Map); _resourceMetadata = ResourceMetadata.DeserializeFrom(dataMap); _mappingLookups = mappingLookups; } @@ -43,6 +44,13 @@ public JToken ApplyMap(Dictionary csvRow) return MapToJsonObject(_resourceMetadata, _mappings, safeCsvRow); } + public JToken ApplyMapForDeleteByIdOperation(Dictionary csvRow) + { + var safeCsvRow = new CsvRow(csvRow); + + return MapToJsonObjectForDeleteById(_mappings, safeCsvRow); + } + private JObject MapToJsonObject(IReadOnlyList nodeMetadatas, IReadOnlyList nodes, CsvRow csvRow) { var result = new JObject(); @@ -79,6 +87,20 @@ private JObject MapToJsonObject(IReadOnlyList nodeMetadatas, I return result; } + private JObject MapToJsonObjectForDeleteById(IReadOnlyList nodes, CsvRow csvRow) + { + var result = new JObject(); + + foreach (var node in nodes) + { + var rawValue = RawValue(node, csvRow); + + result.Add(new JProperty("Id", rawValue)); + } + + return result; + } + private JArray MapToJsonArray(ResourceMetadata arrayItemMetadata, IReadOnlyList nodes, CsvRow csvRow) { var result = new JArray(); diff --git a/DataImport.Web.Tests/Features/DataMaps/AddEditDataMapsTests.cs b/DataImport.Web.Tests/Features/DataMaps/AddEditDataMapsTests.cs index 98c00faa..976d7625 100644 --- a/DataImport.Web.Tests/Features/DataMaps/AddEditDataMapsTests.cs +++ b/DataImport.Web.Tests/Features/DataMaps/AddEditDataMapsTests.cs @@ -140,7 +140,8 @@ public async Task ShouldSuccessfullyAddDataMap() ResourcePath = resource.Path, MapName = mapName, Mappings = mappings, - ColumnHeaders = addForm.ColumnHeaders + ColumnHeaders = addForm.ColumnHeaders, + IsDeleteOperation = addForm.IsDeleteOperation }); response.AssertToast($"Data Map '{mapName}' was created."); @@ -169,7 +170,77 @@ public async Task ShouldSuccessfullyAddDataMap() ApiVersionId = apiVersion.Id, Preprocessors = editForm.Preprocessors, PreprocessorLogMessages = editForm.PreprocessorLogMessages, - ApiServers = editForm.ApiServers + ApiServers = editForm.ApiServers, + IsDeleteOperation = editForm.IsDeleteOperation + }); + } + + [Test] + public async Task ShouldSuccessfullyAddDeleteByIdDataMap() + { + var resource = RandomResource(); + var mapName = SampleString(); + var mappings = (await TrivialMappings(resource)).Take(1).ToArray(); + var apiVersion = Query(d => d.ApiVersions.Single(x => x.Id == resource.ApiVersionId)); + + var dataMapSerializer = new DeleteDataMapSerializer(); + var expectedJsonMap = dataMapSerializer.Serialize(mappings); + var sourceCsvHeaders = new[] { "ColA", "ColB", "ColC" }; + + var addForm = await Send(new AddDataMap.Query { SourceCsvHeaders = sourceCsvHeaders }); + addForm.ColumnHeaders.ShouldMatch("ColA", "ColB", "ColC"); + addForm.FieldsViewModel.DataSources.ShouldMatch( + new SelectListItem { Text = "Select Data Source", Value = "" }, + new SelectListItem { Text = "column", Value = "column" }, + new SelectListItem { Text = "lookup-table", Value = "lookup-table" }, + new SelectListItem { Text = "static", Value = "static" }); + addForm.FieldsViewModel.SourceTables.ShouldMatch(Query(DataMapperFields.MapLookupTablesToViewModel)); + addForm.FieldsViewModel.SourceColumns.ShouldMatch( + new SelectListItem { Text = "Select Source Column", Value = "" }, + new SelectListItem { Text = "ColA", Value = "ColA" }, + new SelectListItem { Text = "ColB", Value = "ColB" }, + new SelectListItem { Text = "ColC", Value = "ColC" }); + addForm.FieldsViewModel.ResourceMetadata.ShouldBeEmpty(); + addForm.FieldsViewModel.Mappings.ShouldBeEmpty(); + + var response = await Send(new AddDataMap.Command + { + ApiVersionId = resource.ApiVersionId, + ResourcePath = resource.Path, + MapName = mapName, + Mappings = mappings, + ColumnHeaders = addForm.ColumnHeaders, + IsDeleteOperation = true + }); + response.AssertToast($"Data Map '{mapName}' was created."); + + var actual = Query(response.DataMapId); + actual.Name.ShouldBe(mapName); + actual.ResourcePath.ShouldBe(resource.Path); + actual.Map.ShouldBe(expectedJsonMap); + actual.Metadata.ShouldBe(resource.Metadata); + actual.CreateDate.ShouldNotBe(null); + actual.UpdateDate.ShouldNotBe(null); + actual.ApiVersionId.ShouldBe(resource.ApiVersionId); + + var editForm = await Send(new EditDataMap.Query { Id = response.DataMapId, SourceCsvHeaders = new string[] { } }); + + editForm.ShouldMatch(new AddEditDataMapViewModel + { + DataMapId = response.DataMapId, + ResourcePath = resource.Path, + ResourceName = resource.ToResourceName(), + MapName = mapName, + ColumnHeaders = sourceCsvHeaders, + FieldsViewModel = editForm.FieldsViewModel, + MetadataIsIncompatible = false, + ApiVersions = editForm.ApiVersions, + ApiVersion = apiVersion.Version, + ApiVersionId = apiVersion.Id, + Preprocessors = editForm.Preprocessors, + PreprocessorLogMessages = editForm.PreprocessorLogMessages, + ApiServers = editForm.ApiServers, + IsDeleteOperation = editForm.IsDeleteOperation }); } @@ -257,6 +328,92 @@ public async Task ShouldSuccessfullyEditDataMap() } } + [Test] + public async Task ShouldSuccessfullyEditDeleteByIdDataMap() + { + // This test deals with editing an empty map to one with a single static + // mapped value, so the only usable resources for these tests are those + // which have at least one mappable property. That *should* be all resources, + // but the simplest way to find a field to map to is to search for resources + // with at least one top-level non-array / non-object property. Most resources + // do meet this requirement. + + var apiVersion = GetDefaultApiVersion(); + + var usableResources = Query(x => x.Resources.Where(r => r.ApiVersionId == apiVersion.Id).ToArray()) + .Where(x => ResourceMetadata.DeserializeFrom(x).Any(IsStaticMappable)) + .ToArray(); + + usableResources.Length.ShouldBeGreaterThan(0); + + foreach (var resource in usableResources) + { + var initialMapName = SampleString(); + var updatedMapName = SampleString(); + var initialMappings = (await TrivialMappings(resource)).Take(1).ToArray(); + var updatedMappings = (await TrivialMappings(resource)).Take(1).ToArray(); + + var columnHeaders = new[] { "ColA", "ColB", "ColC" }; + + var dataMapSerializer = new DeleteDataMapSerializer(); + var expectedJsonMap = dataMapSerializer.Serialize(updatedMappings); + + var response = await Send(new AddDataMap.Command + { + ApiVersionId = resource.ApiVersionId, + ResourcePath = resource.Path, + MapName = initialMapName, + Mappings = initialMappings, + ColumnHeaders = columnHeaders, + IsDeleteOperation = true + }); + + var editForm = await Send(new EditDataMap.Query + { Id = response.DataMapId, SourceCsvHeaders = new string[] { } }); + + editForm.MapName = updatedMapName; + + var toastResponse = await Send(new EditDataMap.Command + { + DataMapId = response.DataMapId, + MapName = editForm.MapName, + Mappings = updatedMappings, + ColumnHeaders = editForm.ColumnHeaders, + IsDeleteOperation = editForm.IsDeleteOperation + }); + toastResponse.AssertToast($"Data Map '{editForm.MapName}' was modified."); + + var actual = Query(response.DataMapId); + actual.Name.ShouldBe(updatedMapName); + actual.ResourcePath.ShouldBe(resource.Path); + actual.Map.ShouldBe(expectedJsonMap); + actual.Metadata.ShouldBe(resource.Metadata); + actual.CreateDate.ShouldNotBe(null); + actual.UpdateDate.ShouldNotBe(null); + actual.IsDeleteOperation.ShouldBe(true); + + var updatedEditForm = await Send(new EditDataMap.Query { Id = response.DataMapId, SourceCsvHeaders = new string[] { } }); + + updatedEditForm.ShouldMatch(new AddEditDataMapViewModel + { + DataMapId = response.DataMapId, + ResourcePath = resource.Path, + ResourceName = resource.ToResourceName(), + MapName = updatedMapName, + ColumnHeaders = columnHeaders, + FieldsViewModel = updatedEditForm.FieldsViewModel, + MetadataIsIncompatible = false, + ApiVersions = updatedEditForm.ApiVersions, + ApiVersion = apiVersion.Version, + ApiVersionId = apiVersion.Id, + ApiServers = updatedEditForm.ApiServers, + PreprocessorLogMessages = updatedEditForm.PreprocessorLogMessages, + Preprocessors = updatedEditForm.Preprocessors, + IsDeleteOperation = updatedEditForm.IsDeleteOperation + }); + } + } + [Test] public async Task ShouldRequireAttributeForPreprocessor() { diff --git a/DataImport.Web.Tests/SetUpFixture.cs b/DataImport.Web.Tests/SetUpFixture.cs index 3eab6687..e74134ab 100644 --- a/DataImport.Web.Tests/SetUpFixture.cs +++ b/DataImport.Web.Tests/SetUpFixture.cs @@ -3,14 +3,12 @@ // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. -using System; using DataImport.Models; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using NUnit.Framework; using System.Reflection; using System.Threading.Tasks; -using Microsoft.Extensions.Configuration; using Serilog; using static DataImport.Web.Tests.Testing; @@ -33,8 +31,8 @@ public async Task GlobalSetUp() using (var context = Testing.Services.GetRequiredService()) { - context.Database.EnsureDeleted(); - context.Database.Migrate(); + await context.Database.EnsureDeletedAsync(); + await context.Database.MigrateAsync(); } Log.Information(Assembly.GetExecutingAssembly().GetName().Name + " Starting"); diff --git a/DataImport.Web/Features/DataMaps/AddDataMap.cs b/DataImport.Web/Features/DataMaps/AddDataMap.cs index 499291c3..90a79db7 100644 --- a/DataImport.Web/Features/DataMaps/AddDataMap.cs +++ b/DataImport.Web/Features/DataMaps/AddDataMap.cs @@ -69,6 +69,7 @@ public class Command : IRequest public string[] ColumnHeaders { get; set; } public int? PreprocessorId { get; set; } public string Attribute { get; set; } + public bool IsDeleteOperation { get; set; } } public class Response : ToastResponse @@ -123,7 +124,9 @@ protected override Response Handle(Command request) { Name = request.MapName, ResourcePath = resource.Path, - Map = dataMapSerializer.Serialize(request.Mappings), + Map = request.IsDeleteOperation + ? new DeleteDataMapSerializer().Serialize(request.Mappings) + : new DataMapSerializer(resource).Serialize(request.Mappings), Metadata = resource.Metadata, CreateDate = DateTimeOffset.Now, UpdateDate = DateTimeOffset.Now, @@ -133,7 +136,8 @@ protected override Response Handle(Command request) : JsonConvert.SerializeObject(request.ColumnHeaders), ApiVersionId = request.ApiVersionId, FileProcessorScriptId = request.PreprocessorId, - Attribute = request.Attribute + Attribute = request.Attribute, + IsDeleteOperation = request.IsDeleteOperation }; _database.DataMaps.Add(dataMap); diff --git a/DataImport.Web/Features/DataMaps/AddEdit.cshtml b/DataImport.Web/Features/DataMaps/AddEdit.cshtml index 0406da80..c7d27218 100644 --- a/DataImport.Web/Features/DataMaps/AddEdit.cshtml +++ b/DataImport.Web/Features/DataMaps/AddEdit.cshtml @@ -127,6 +127,8 @@ See the LICENSE and NOTICES files in the project root for more information. + @Html.Input(m => m.IsDeleteOperation) +
} +
+

Delete by Id

+

This data map will delete the rows from the selected ODS / API instance based on the unique identifier given in the selected resource as selected above. It will perform a lookup by these values, and then send a DELETE command to the ODS API to remove the data from the instance. Please use this function with caution and ensure any critical data is backed up before using this feature.

+
+ @if (Model.PreprocessorLogMessages?.Count > 0) {
@@ -257,15 +264,28 @@ See the LICENSE and NOTICES files in the project root for more information. { @Html.ValidationSummary(true, "", new { @class = "text-danger" }) + @if (!Model.IsDeleteOperation) + {
@Html.Label("CSV Fields")
@await Html.PartialAsync("_PartialDataMapperFields", Model.FieldsViewModel)
- @Html.Hidden("ColumnHeaders", JsonConvert.SerializeObject(Model.ColumnHeaders)) - @Html.SubmitButton("Save Map", new { id = "btnSubmit" })} - } + + } + else + { +
+ @Html.Label("CSV Fields") +
+
+ @await Html.PartialAsync("_PartialDataMapperDeleteByIdFields", Model.FieldsViewModel) +
+ } + @Html.Hidden("ColumnHeaders", JsonConvert.SerializeObject(Model.ColumnHeaders)) + @Html.SubmitButton("Save Map", new { id = "btnSubmit" })} +} @section scripts{