From 2a516bf15bc863ffbf3b5f92b0d6952b068e9911 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 25 Sep 2024 14:05:03 +0100 Subject: [PATCH 1/7] add changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c4ac89757..172ecdb1dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [8.4.0] - Unreleased + ## [8.3.1] - Unreleased - Improve Performance of regenerating problems with child providers From 53596b0457088e55eac543d8f70b5fe4a5849e1a Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 26 Sep 2024 10:21:29 +0100 Subject: [PATCH 2/7] Bugfix/rdmp 253 filter ordering (#2007) * add start * partial * tidy up code * filter updates * update db migrations * update correct db * update changelog * add base creation sql * fix create * tidy up --- CHANGELOG.md | 3 + .../Data/Aggregation/AggregateFilter.cs | 6 +- Rdmp.Core/Curation/Data/ConcreteFilter.cs | 5 +- Rdmp.Core/Curation/Data/ExtractionFilter.cs | 6 +- .../SpontaneouslyInventedFilter.cs | 5 +- .../Data/DeployedExtractionFilter.cs | 6 +- .../CreateCatalogue.sql | 2 + .../up/087_AddAggregateFilterOrdering.sql | 13 ++++ .../CreateDataExportManager.sql | Bin 207702 -> 207772 bytes .../up/026_AddFilterOrder.sql | 8 +++ Rdmp.Core/Rdmp.Core.csproj | 4 ++ .../ExecuteCommandReorderFilter.cs | 62 ++++++++++++++++++ ...poseExecutionWhenTargetIsConcreteFilter.cs | 18 +++-- SharedAssemblyInfo.cs | 6 +- 14 files changed, 131 insertions(+), 13 deletions(-) create mode 100644 Rdmp.Core/Databases/CatalogueDatabase/up/087_AddAggregateFilterOrdering.sql create mode 100644 Rdmp.Core/Databases/DataExportDatabase/up/026_AddFilterOrder.sql create mode 100644 Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReorderFilter.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 172ecdb1dd..3bd8a30350 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ + # Changelog All notable changes to this project will be documented in this file. @@ -7,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [8.4.0] - Unreleased +- Add Ordering to Filters + ## [8.3.1] - Unreleased - Improve Performance of regenerating problems with child providers diff --git a/Rdmp.Core/Curation/Data/Aggregation/AggregateFilter.cs b/Rdmp.Core/Curation/Data/Aggregation/AggregateFilter.cs index 7b23404346..307a7ef8aa 100644 --- a/Rdmp.Core/Curation/Data/Aggregation/AggregateFilter.cs +++ b/Rdmp.Core/Curation/Data/Aggregation/AggregateFilter.cs @@ -1,4 +1,4 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2024 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. @@ -35,6 +35,7 @@ public class AggregateFilter : ConcreteFilter, IDisableable private int? _clonedFromExtractionFilterID; private int? _associatedColumnInfoID; private bool _isDisabled; + private int _order; /// public override int? ClonedFromExtractionFilter_ID @@ -90,6 +91,8 @@ public IEnumerable AggregateFilterParameters ? Repository.GetObjectByID(FilterContainer_ID.Value) : null; + public override int Order { get => _order; set => SetField(ref _order, value); } + #endregion public AggregateFilter() @@ -121,6 +124,7 @@ internal AggregateFilter(ICatalogueRepository repository, DbDataReader r) : base Name = r["Name"] as string; IsMandatory = (bool)r["IsMandatory"]; ClonedFromExtractionFilter_ID = ObjectToNullableInt(r["ClonedFromExtractionFilter_ID"]); + Order = int.Parse(r["Order"].ToString()); var associatedColumnInfo_ID = r["AssociatedColumnInfo_ID"]; if (associatedColumnInfo_ID != DBNull.Value) diff --git a/Rdmp.Core/Curation/Data/ConcreteFilter.cs b/Rdmp.Core/Curation/Data/ConcreteFilter.cs index c29ebbdeca..b8fe1f49de 100644 --- a/Rdmp.Core/Curation/Data/ConcreteFilter.cs +++ b/Rdmp.Core/Curation/Data/ConcreteFilter.cs @@ -1,4 +1,4 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2024 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. @@ -25,7 +25,7 @@ namespace Rdmp.Core.Curation.Data; /// ConcreteFilter is used to provide UI editing of an IFilter without having to add persistence / DatabaseEntity logic to IFilter (which would break /// SpontaneouslyInventedFilters) /// -public abstract class ConcreteFilter : DatabaseEntity, IFilter, ICheckable +public abstract class ConcreteFilter : DatabaseEntity, IFilter, ICheckable, IOrderable { /// protected ConcreteFilter(IRepository repository, DbDataReader r) : base(repository, r) @@ -100,6 +100,7 @@ public bool IsMandatory /// [NoMappingToDatabase] public abstract IContainer FilterContainer { get; } + public abstract int Order { get; set; } #endregion diff --git a/Rdmp.Core/Curation/Data/ExtractionFilter.cs b/Rdmp.Core/Curation/Data/ExtractionFilter.cs index f6ce6f8d4c..febd12b629 100644 --- a/Rdmp.Core/Curation/Data/ExtractionFilter.cs +++ b/Rdmp.Core/Curation/Data/ExtractionFilter.cs @@ -1,4 +1,4 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2024 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. @@ -40,6 +40,7 @@ public class ExtractionFilter : ConcreteFilter, IHasDependencies, IInjectKnown _knownExtractionFilterParameterSets; + private int _order; /// /// The column in the which is best/most associated with this filter. A filter can query any column in any of the table(s) under @@ -133,6 +134,8 @@ internal ExtractionFilter(ICatalogueRepository repository, DbDataReader r) Description = r["Description"] as string; Name = r["Name"] as string; IsMandatory = (bool)r["IsMandatory"]; + Order = int.Parse(r["Order"].ToString()); + ClearAllInjections(); } @@ -154,6 +157,7 @@ public override int? ClonedFromExtractionFilter_ID set => throw new NotSupportedException( "ClonedFromExtractionFilter_ID is only supported on lower level filters e.g. DeployedExtractionFilter and AggregateFilter"); } + public override int Order { get => _order; set => SetField(ref _order,value); } /// public IHasDependencies[] GetObjectsThisDependsOn() diff --git a/Rdmp.Core/Curation/Data/Spontaneous/SpontaneouslyInventedFilter.cs b/Rdmp.Core/Curation/Data/Spontaneous/SpontaneouslyInventedFilter.cs index 8bbadc4f85..dd8458426f 100644 --- a/Rdmp.Core/Curation/Data/Spontaneous/SpontaneouslyInventedFilter.cs +++ b/Rdmp.Core/Curation/Data/Spontaneous/SpontaneouslyInventedFilter.cs @@ -1,4 +1,4 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2024 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. @@ -22,6 +22,7 @@ public class SpontaneouslyInventedFilter : ConcreteFilter { private readonly MemoryCatalogueRepository _repo; private readonly ISqlParameter[] _filterParametersIfAny; + private int _order =0; /// /// Creates a new temporary (unsaveable) filter in the given memory @@ -68,6 +69,8 @@ public SpontaneouslyInventedFilter(MemoryCatalogueRepository repo, IFilter copyF ? _repo.GetObjectByID(FilterContainer_ID.Value) : null; + public override int Order { get => _order; set => SetField(ref _order, value); } + public override ColumnInfo GetColumnInfoIfExists() => null; public override IFilterFactory GetFilterFactory() => null; diff --git a/Rdmp.Core/DataExport/Data/DeployedExtractionFilter.cs b/Rdmp.Core/DataExport/Data/DeployedExtractionFilter.cs index d1e7960ade..122eece307 100644 --- a/Rdmp.Core/DataExport/Data/DeployedExtractionFilter.cs +++ b/Rdmp.Core/DataExport/Data/DeployedExtractionFilter.cs @@ -1,4 +1,4 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2024 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. @@ -36,6 +36,7 @@ public class DeployedExtractionFilter : ConcreteFilter private int? _clonedFromExtractionFilterID; private int? _filterContainerID; + private int _order; /// public override int? ClonedFromExtractionFilter_ID @@ -69,6 +70,8 @@ public override int? FilterContainer_ID ? Repository.GetObjectByID(FilterContainer_ID.Value) : null; + public override int Order { get => _order; set => SetField(ref _order, value); } + #endregion /// @@ -138,6 +141,7 @@ internal DeployedExtractionFilter(IDataExportRepository repository, DbDataReader FilterContainer_ID = null; ClonedFromExtractionFilter_ID = ObjectToNullableInt(r["ClonedFromExtractionFilter_ID"]); + Order = int.Parse(r["Order"].ToString()); } /// diff --git a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql index e794b1e35c..418fcf7e9d 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql @@ -141,6 +141,7 @@ CREATE TABLE [dbo].[AggregateFilter]( [AssociatedColumnInfo_ID] [int] NULL, [ID] [int] IDENTITY(1,1) NOT NULL, [SoftwareVersion] [nvarchar](50) NOT NULL, + [Order] [int] NOT NULL DEFAULT 0 CONSTRAINT [PK_AggregateFilter] PRIMARY KEY CLUSTERED ( [ID] ASC @@ -464,6 +465,7 @@ CREATE TABLE [dbo].[ExtractionFilter]( [Name] [varchar](100) NOT NULL, [IsMandatory] [bit] NOT NULL, [SoftwareVersion] [nvarchar](50) NOT NULL, + [Order] [int] NOT NULL DEFAULT 0 CONSTRAINT [PK_ExtractionFilter] PRIMARY KEY CLUSTERED ( [ID] ASC diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/087_AddAggregateFilterOrdering.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/087_AddAggregateFilterOrdering.sql new file mode 100644 index 0000000000..0d3e473416 --- /dev/null +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/087_AddAggregateFilterOrdering.sql @@ -0,0 +1,13 @@ +----Version: 8.4.0 +----Description: Add Order to Aggregate Filters + +if not exists (select 1 from sys.columns where name = 'Order' and OBJECT_NAME(object_id) = 'AggregateFilter') +BEGIN +ALTER TABLE [dbo].[AggregateFilter] +ADD [Order] [int] NOT NULL DEFAULT 0 WITH VALUES +END +if not exists (select 1 from sys.columns where name = 'Order' and OBJECT_NAME(object_id) = 'ExtractionFilter') +BEGIN +ALTER TABLE [dbo].[ExtractionFilter] +ADD [Order] [int] NOT NULL DEFAULT 0 WITH VALUES +END \ No newline at end of file diff --git a/Rdmp.Core/Databases/DataExportDatabase/runAfterCreateDatabase/CreateDataExportManager.sql b/Rdmp.Core/Databases/DataExportDatabase/runAfterCreateDatabase/CreateDataExportManager.sql index 15f75c2cc478241235068c0ad4aa43f49a778571..ac1c03a92acf841ff12f997ebad6538dc418bdb8 100644 GIT binary patch delta 56 zcmcb1jc3kvo(&l?lM7tgCND6Sn%uz8H#sO-h|yp&BdhAV!Z delta 30 lcmbPpo#)y$o(&l?lM})fn$O9!pOaw(Vy5lqWSGzK0RYrq45$DA diff --git a/Rdmp.Core/Databases/DataExportDatabase/up/026_AddFilterOrder.sql b/Rdmp.Core/Databases/DataExportDatabase/up/026_AddFilterOrder.sql new file mode 100644 index 0000000000..1181d65c9b --- /dev/null +++ b/Rdmp.Core/Databases/DataExportDatabase/up/026_AddFilterOrder.sql @@ -0,0 +1,8 @@ +----Version: 8.4.0 +----Description: Add Order to Aggregate Filters + +if not exists (select 1 from sys.columns where name = 'Order' and OBJECT_NAME(object_id) = 'DeployedExtractionFilter') +BEGIN +ALTER TABLE [dbo].[DeployedExtractionFilter] +ADD [Order] [int] NOT NULL DEFAULT 0 WITH VALUES +END \ No newline at end of file diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index 449cc6abdb..3f2209d40d 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -129,6 +129,7 @@ + @@ -157,6 +158,7 @@ + @@ -255,6 +257,7 @@ + @@ -297,6 +300,7 @@ + diff --git a/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReorderFilter.cs b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReorderFilter.cs new file mode 100644 index 0000000000..e6483fa5a1 --- /dev/null +++ b/Rdmp.UI/CommandExecution/AtomicCommands/ExecuteCommandReorderFilter.cs @@ -0,0 +1,62 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.Curation.Data; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using Rdmp.UI.ItemActivation; +using System; +using System.Linq; + +namespace Rdmp.UI.CommandExecution.AtomicCommands; + +public class ExecuteCommandReorderFilter : BasicUICommandExecution +{ + private ConcreteFilter _source; + private ConcreteFilter _target; + private InsertOption _insertOption; + + public ExecuteCommandReorderFilter(IActivateItems activator, ConcreteFilter source, ConcreteFilter destination, InsertOption insertOption) : base(activator) + { + _source = source; + _target = destination; + _insertOption = insertOption; + if (_source.FilterContainer_ID is null || _target.FilterContainer_ID is null) + { + SetImpossible("Both filters must exist within some container in order to be orderable"); + } + if (_source.FilterContainer_ID != _target.FilterContainer_ID) + { + SetImpossible("Cannot reorder filters as they do not share a parent"); + } + } + + public override void Execute() + { + var order = _target.Order; + + var filters = _target.FilterContainer.GetFilters().Where(f => f is ConcreteFilter).Select(f => (ConcreteFilter)f).ToArray(); + Array.Sort( + filters, + delegate (ConcreteFilter a, ConcreteFilter b) { return a.Order.CompareTo(b.Order); } + ); + if (!filters.All(c => c.Order != order)) + { + foreach (var orderable in filters) + { + if (orderable.Order < order) + orderable.Order--; + else if (orderable.Order > order) + orderable.Order++; + else //collision on order + orderable.Order += _insertOption == InsertOption.InsertAbove ? 1 : -1; + ((ISaveable)orderable).SaveToDatabase(); + } + } + _source.Order = order; + _source.SaveToDatabase(); + Publish(_target.FilterContainer); + } +} diff --git a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsConcreteFilter.cs b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsConcreteFilter.cs index 93f6fb3c1c..98d290b1b0 100644 --- a/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsConcreteFilter.cs +++ b/Rdmp.UI/CommandExecution/Proposals/ProposeExecutionWhenTargetIsConcreteFilter.cs @@ -5,7 +5,9 @@ // You should have received a copy of the GNU General Public License along with RDMP. If not, see . using Rdmp.Core.CommandExecution; +using Rdmp.Core.CommandExecution.Combining; using Rdmp.Core.Curation.Data; +using Rdmp.UI.CommandExecution.AtomicCommands; using Rdmp.UI.ExtractionUIs.FilterUIs; using Rdmp.UI.ItemActivation; @@ -24,8 +26,16 @@ public override void Activate(ConcreteFilter target) ItemActivator.Activate(target); } - public override ICommandExecution ProposeExecution(ICombineToMakeCommand cmd, ConcreteFilter target, - InsertOption insertOption = InsertOption.Default) => - //currently nothing can be dropped onto a filter - null; + public override ICommandExecution ProposeExecution(ICombineToMakeCommand cmd, ConcreteFilter targetFilter, + InsertOption insertOption = InsertOption.Default) + { + return cmd switch + { + FilterCombineable sourceFilterCommand => + !sourceFilterCommand.Filter.Equals(targetFilter) && sourceFilterCommand.Filter is ConcreteFilter && sourceFilterCommand.Filter.FilterContainer_ID == targetFilter.FilterContainer_ID ? + new ExecuteCommandReorderFilter(ItemActivator, (ConcreteFilter)sourceFilterCommand.Filter, targetFilter, insertOption) + : null, + _ => null + }; + } } \ No newline at end of file diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 559a541653..dd904c2b97 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("8.3.0")] -[assembly: AssemblyFileVersion("8.3.0")] -[assembly: AssemblyInformationalVersion("8.3.0")] \ No newline at end of file +[assembly: AssemblyVersion("8.4.0")] +[assembly: AssemblyFileVersion("8.4.0")] +[assembly: AssemblyInformationalVersion("8.4.0")] \ No newline at end of file From 7b93df0aafeb91eff934a8bb1e0aa4a9a3aa4892 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 3 Oct 2024 13:14:52 +0100 Subject: [PATCH 3/7] add order to insert --- .../OverwriteMigrationStrategy.cs | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs b/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs index d7b696231e..4294c77820 100644 --- a/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs +++ b/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs @@ -6,12 +6,15 @@ using System; using System.Collections.Generic; +using System.Data; using System.Linq; using System.Text; using FAnsi; using FAnsi.Connections; using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; +using MongoDB.Driver; +using NPOI.SS.Formula.Functions; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataLoad.Engine.Job; using Rdmp.Core.DataLoad.Triggers; @@ -100,6 +103,38 @@ CrossDatabaseMergeCommandTo..ToTable.Age is null sbInsert.AppendLine( $"{columnsToMigrate.DestinationTable.GetFullyQualifiedName()}.{syntax.EnsureWrapped(columnsToMigrate.PrimaryKeys.First().GetRuntimeName())} IS NULL"); + if (columnsToMigrate.PrimaryKeys.Any())//should have some on/off switch + { + //add order by pk values + var orderSQL = $@" +SELECT KU.ORDINAL_POSITION AS ORDINAL_POSITION + , COLUMN_NAME, TC.CONSTRAINT_NAME as name, is_descending_key +FROM + INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS TC +INNER JOIN + INFORMATION_SCHEMA.KEY_COLUMN_USAGE AS KU ON TC.CONSTRAINT_TYPE = 'PRIMARY KEY' + AND TC.CONSTRAINT_NAME = KU.CONSTRAINT_NAME + AND KU.TABLE_NAME = '{columnsToMigrate.DestinationTable.GetRuntimeName()}' +LEFT JOIN(select COL_NAME(ic.object_id,ic.column_id) as cn, i.name, is_descending_key FROM sys.indexes i +INNER JOIN sys.data_spaces ds ON i.data_space_id = ds.data_space_id +INNER JOIN sys.index_columns ic ON i.object_id = ic.object_id AND i.index_id = ic.index_id +where is_primary_key=1) SY ON SY.cn = KU.COLUMN_NAME AND SY.name = KU.CONSTRAINT_NAME +ORDER BY ORDINAL_POSITION asc + "; + var dt = new DataTable(); + dt.BeginLoadData(); + + using (var orderCmd = server.GetCommand(orderSQL, _managedConnection)) + { + orderCmd.CommandTimeout = Timeout; + using var da = server.GetDataAdapter(orderCmd); + da.Fill(dt); + } + var orderList = String.Join(", ", dt.AsEnumerable().Select(row => $"{row[1]} {((bool)row[3]? "DESC":"ASC")}")); + var orderString = $"ORDER BY {orderList}"; + sbInsert.Append(orderString); + } + //right at the end of the SELECT if (columnsToMigrate.DestinationTable.Database.Server.DatabaseType == DatabaseType.MySql) sbInsert.Append(" FOR UPDATE"); From 60ee99b2abaf1100c17cd2d84f7ef59a3725fd25 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 3 Oct 2024 15:01:37 +0100 Subject: [PATCH 4/7] add lmd update --- .../CommandExecution/AtomicCommandFactory.cs | 4 +++ ...ggleInsertOrderingPrefixForLoadMetadata.cs | 30 +++++++++++++++++++ .../Curation/Data/DataLoad/ILoadMetadata.cs | 6 ++++ .../Curation/Data/DataLoad/LoadMetadata.cs | 8 +++++ .../OverwriteMigrationStrategy.cs | 1 - .../CreateCatalogue.sql | 1 + .../up/087_AddDataLoadInsertOrder.sql | 8 +++++ Rdmp.Core/Rdmp.Core.csproj | 2 ++ SharedAssemblyInfo.cs | 6 ++-- 9 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandToggleInsertOrderingPrefixForLoadMetadata.cs create mode 100644 Rdmp.Core/Databases/CatalogueDatabase/up/087_AddDataLoadInsertOrder.sql diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index e2e35c627b..06cf57fddb 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -511,6 +511,10 @@ public IEnumerable CreateCommands(object o) { OverrideCommandName=$"{reservedTest} Reserved Prefix Columns" }; + yield return new ExecuteCommandToggleInsertOrderingPrefixForLoadMetadata(lmd) + { + OverrideCommandName = $"{(lmd.OrderInsertsByPrimaryKey ? "Don't":"")} Sort Index Items by PK before Insert" + }; yield return new ExecuteCommandSetGlobalDleIgnorePattern(_activator) { SuggestedCategory = Advanced }; yield return new ExecuteCommandSetIgnoredColumns(_activator, lmd) { SuggestedCategory = Advanced }; diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandToggleInsertOrderingPrefixForLoadMetadata.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandToggleInsertOrderingPrefixForLoadMetadata.cs new file mode 100644 index 0000000000..536020258d --- /dev/null +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandToggleInsertOrderingPrefixForLoadMetadata.cs @@ -0,0 +1,30 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.DataLoad; + +namespace Rdmp.Core.CommandExecution.AtomicCommands; + +/// +/// Toggles the LoadMetadata's ability to sort index items by pk before inserting +/// +public class ExecuteCommandToggleInsertOrderingPrefixForLoadMetadata : BasicCommandExecution +{ + private LoadMetadata _loadMetadata; + public ExecuteCommandToggleInsertOrderingPrefixForLoadMetadata([DemandsInitialization("The LoadMetadata to update")] LoadMetadata loadMetadata) + { + + _loadMetadata = loadMetadata; + } + + public override void Execute() + { + base.Execute(); + _loadMetadata.OrderInsertsByPrimaryKey = !_loadMetadata.OrderInsertsByPrimaryKey; + _loadMetadata.SaveToDatabase(); + } +} diff --git a/Rdmp.Core/Curation/Data/DataLoad/ILoadMetadata.cs b/Rdmp.Core/Curation/Data/DataLoad/ILoadMetadata.cs index 89bc7eace3..a906cb5bdd 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/ILoadMetadata.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/ILoadMetadata.cs @@ -61,6 +61,12 @@ public interface ILoadMetadata : INamed, ILoggedActivityRootObject /// bool AllowReservedPrefix { get; } + + /// + /// Optional - Allows Data Load inserts to be ordered by the PK for efficiency when using clustered PKS. Requires access to sys.indexes, INFORMATION_SCHEMA.TABLE_CONSTRAINTS and INFORMATION_SCHEMA.KEY_COLUMN_USAGE + /// + bool OrderInsertsByPrimaryKey { get; } + /// /// List of all the user configured steps in a data load. For example you could have 2 ProcessTasks, one that downloads files from an FTP server and one that loads RAW. /// diff --git a/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs b/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs index 2b02f72d40..1fea5dbf8d 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs @@ -61,6 +61,7 @@ public class LoadMetadata : DatabaseEntity, ILoadMetadata, IHasDependencies, IHa private string _folder; private DateTime? _lastLoadTime; private bool _allowReservedPrefix; + private bool _orderInsertsByPK; public string DefaultForLoadingPath = Path.Combine("Data", "ForLoading"); public string DefaultForArchivingPath = Path.Combine("Data", "ForArchiving"); @@ -91,6 +92,13 @@ public bool AllowReservedPrefix set => SetField(ref _allowReservedPrefix, value); } + /// + public bool OrderInsertsByPrimaryKey + { + get => _orderInsertsByPK; + set => SetField(ref _orderInsertsByPK, value); + } + /// public string LocationOfForLoadingDirectory { diff --git a/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs b/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs index 4294c77820..26221ef6b5 100644 --- a/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs +++ b/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs @@ -105,7 +105,6 @@ CrossDatabaseMergeCommandTo..ToTable.Age is null if (columnsToMigrate.PrimaryKeys.Any())//should have some on/off switch { - //add order by pk values var orderSQL = $@" SELECT KU.ORDINAL_POSITION AS ORDINAL_POSITION , COLUMN_NAME, TC.CONSTRAINT_NAME as name, is_descending_key diff --git a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql index e794b1e35c..f2999ad92c 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql @@ -599,6 +599,7 @@ CREATE TABLE [dbo].[LoadMetadata]( [CacheArchiveType] [int] NOT NULL, [SoftwareVersion] [nvarchar](50) NOT NULL, [AllowReservedPrefix] [bit] NOT NULL default 0, + [OrderInsertsByPrimaryKey] [bit] NOT NULL default 0, CONSTRAINT [PK_LoadMetadata] PRIMARY KEY CLUSTERED ( [ID] ASC diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/087_AddDataLoadInsertOrder.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/087_AddDataLoadInsertOrder.sql new file mode 100644 index 0000000000..8c46210b61 --- /dev/null +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/087_AddDataLoadInsertOrder.sql @@ -0,0 +1,8 @@ +--Version: 8.4.0 +--Description: Add ability to allow data loads to order inserts based on promary key + + if not exists (select 1 from sys.columns where name = 'OrderInsertsByPrimaryKey' and OBJECT_NAME(object_id) = 'LoadMetadata') + BEGIN + ALTER TABLE [dbo].[LoadMetadata] + ADD OrderInsertsByPrimaryKey [bit] NOT NULL DEFAULT 0 WITH VALUES + END \ No newline at end of file diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index 449cc6abdb..1a8732f893 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -129,6 +129,7 @@ + @@ -255,6 +256,7 @@ + diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 559a541653..dd904c2b97 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("8.3.0")] -[assembly: AssemblyFileVersion("8.3.0")] -[assembly: AssemblyInformationalVersion("8.3.0")] \ No newline at end of file +[assembly: AssemblyVersion("8.4.0")] +[assembly: AssemblyFileVersion("8.4.0")] +[assembly: AssemblyInformationalVersion("8.4.0")] \ No newline at end of file From 467c7e4cc28fc0ca0d612f027da89edf6609f4c2 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 3 Oct 2024 15:04:52 +0100 Subject: [PATCH 5/7] move to advanced --- Rdmp.Core/CommandExecution/AtomicCommandFactory.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index 06cf57fddb..682eb28eb5 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -509,11 +509,13 @@ public IEnumerable CreateCommands(object o) var reservedTest = lmd.AllowReservedPrefix ? "Drop" : "Allow"; yield return new ExecuteCommandToggleAllowReservedPrefixForLoadMetadata(lmd) { - OverrideCommandName=$"{reservedTest} Reserved Prefix Columns" + OverrideCommandName = $"{reservedTest} Reserved Prefix Columns", + SuggestedCategory = "Advanced" }; yield return new ExecuteCommandToggleInsertOrderingPrefixForLoadMetadata(lmd) { - OverrideCommandName = $"{(lmd.OrderInsertsByPrimaryKey ? "Don't":"")} Sort Index Items by PK before Insert" + OverrideCommandName = $"{(lmd.OrderInsertsByPrimaryKey ? "Don't" : "")} Sort Index Items by PK before Insert", + SuggestedCategory = "Advanced" }; yield return new ExecuteCommandSetGlobalDleIgnorePattern(_activator) { SuggestedCategory = Advanced }; From 7b68572ab274f0dcf5d129be8e1e457e6c41d725 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 3 Oct 2024 16:17:21 +0100 Subject: [PATCH 6/7] add check --- .../Migration/QueryBuilding/OverwriteMigrationStrategy.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs b/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs index 26221ef6b5..510abab1ac 100644 --- a/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs +++ b/Rdmp.Core/DataLoad/Engine/Migration/QueryBuilding/OverwriteMigrationStrategy.cs @@ -103,7 +103,7 @@ CrossDatabaseMergeCommandTo..ToTable.Age is null sbInsert.AppendLine( $"{columnsToMigrate.DestinationTable.GetFullyQualifiedName()}.{syntax.EnsureWrapped(columnsToMigrate.PrimaryKeys.First().GetRuntimeName())} IS NULL"); - if (columnsToMigrate.PrimaryKeys.Any())//should have some on/off switch + if (job.LoadMetadata.OrderInsertsByPrimaryKey && columnsToMigrate.PrimaryKeys.Any()) { var orderSQL = $@" SELECT KU.ORDINAL_POSITION AS ORDINAL_POSITION From 56d4c9d3d161e307e24d5734f799f89424ed444b Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 8 Oct 2024 07:49:39 +0100 Subject: [PATCH 7/7] rename --- CHANGELOG.md | 2 +- Rdmp.Core/CommandExecution/AtomicCommandFactory.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64be6778fd..b266dadf20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [8.4.0] - Unreleased - Add Ordering to Filters -- Add abaility to order inserts during Data Load +- Add ability to optimise inserts for sequential keys during Data Load ## [8.3.1] - Unreleased diff --git a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs index 682eb28eb5..9c5b9e6d8e 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommandFactory.cs @@ -514,7 +514,7 @@ public IEnumerable CreateCommands(object o) }; yield return new ExecuteCommandToggleInsertOrderingPrefixForLoadMetadata(lmd) { - OverrideCommandName = $"{(lmd.OrderInsertsByPrimaryKey ? "Don't" : "")} Sort Index Items by PK before Insert", + OverrideCommandName = $"{(lmd.OrderInsertsByPrimaryKey ? "Don't" : "")} Optimise Inserts for Sequential Keys", SuggestedCategory = "Advanced" };