Release: 8.4.0

Application/ResearchDataManagementPlatform/WindowManagement/TopBar/RDMPTaskBarUI.cs

@@ -51,11 +51,11 @@
         btnTables.Image = CatalogueIcons.TableInfo.ImageToBitmap();
         btnTables.BackgroundImage = BackColorProvider.GetBackgroundImage(btnTables.Size, RDMPCollection.Tables);
 
-        btnDataSets.Image = CatalogueIcons.Dataset.ImageToBitmap();
-        btnDataSets.BackgroundImage = BackColorProvider.GetBackgroundImage(btnDataSets.Size, RDMPCollection.Datasets);
+        btnConfiguration.Image = FamFamFamIcons.pencil_go.ImageToBitmap();
+        btnConfiguration.BackgroundImage = BackColorProvider.GetBackgroundImage(btnConfiguration.Size, RDMPCollection.Datasets);
 
         btnLoads.Image = CatalogueIcons.LoadMetadata.ImageToBitmap();
-        btnLoads.BackgroundImage = BackColorProvider.GetBackgroundImage(btnDataSets.Size, RDMPCollection.DataLoad);
+        btnLoads.BackgroundImage = BackColorProvider.GetBackgroundImage(btnConfiguration.Size, RDMPCollection.DataLoad);
 
         btnFavourites.Image = CatalogueIcons.Favourite.ImageToBitmap();
         btnDeleteLayout.Image = FamFamFamIcons.delete.ImageToBitmap();
@@ -111,7 +111,7 @@
             btnTables.ToolTipText = "Advanced features e.g. logging, credentials, dashboards etc";
             btnLoads.ToolTipText = "Load configurations for reading data into your databases";
             btnFavourites.ToolTipText = "Collection of all objects that you have favourited";
-            btnDataSets.ToolTipText = "All external datasets that have been configured for use in RDMP";
+            btnConfiguration.ToolTipText = "All external datasets that have been configured for use in RDMP";
         }
         catch (Exception e)
         {
@@ -190,8 +190,8 @@
             collectionToToggle = RDMPCollection.SavedCohorts;
         else if (button == btnFavourites)
             collectionToToggle = RDMPCollection.Favourites;
-        else if (button == btnDataSets)
-            collectionToToggle = RDMPCollection.Datasets;
+        else if (button == btnConfiguration)
+            collectionToToggle = RDMPCollection.Configurations;
         else
             throw new ArgumentOutOfRangeException(nameof(button));
 

Application/ResearchDataManagementPlatform/WindowManagement/TopBar/RDMPTaskBarUI.resx

@@ -178,7 +178,7 @@
         TgDQASA1MVpwzwAAAABJRU5ErkJggg==
 </value>
   </data>
-  <data name="btnDataSets.Image" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
+  <data name="btnConfiguration.Image" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
     <value>
         iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8
         YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAAIDSURBVDhPpZLrS5NhGMb3j4SWh0oRQVExD4gonkDpg4hG

Application/ResearchDataManagementPlatform/WindowManagement/WindowManager.cs

@@ -148,10 +148,10 @@ public PersistableToolboxDockContent Create(RDMPCollection collectionToCreate,
                     Image.Load<Rgba32>(CatalogueIcons.Favourite));
                 break;
 
-            case RDMPCollection.Datasets:
-                collection = new DatasetsCollectionUI();
-                toReturn = Show(RDMPCollection.Datasets, collection, "Datasets",
-                    Image.Load<Rgba32>(CatalogueIcons.Dataset));
+            case RDMPCollection.Configurations:
+                collection = new ConfigurationsCollectionUI();
+                toReturn = Show(RDMPCollection.Configurations, collection, "Configurations",
+                     Image.Load<Rgba32>(FamFamFamIcons.pencil_go));
                 break;
 
             default: throw new ArgumentOutOfRangeException(nameof(collectionToCreate));

CHANGELOG.md

@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [8.4.0] - Unreleased
+
+- Add Ordering to Filters
+- [MSSQL ONLY] Add ability to perform Regex redactions on data loads and existing catalogues
+- Add overview page for Catalogues
+- Add RAW Table Date Column Override for Delta Loads
+- Fix Delta Load off by one issue
+- Update Migration strategy to account for all Primary Keys when moving from staging -> live
+- Fix UI issue with viewing cross-database SQL results
+- Add UI Steps to deprecate old cohorts when importing a cohort
+- Add instance setting to enable and disable Yes/No to all
+
 ## [8.3.1] - 2024-10-22
 
 - Improve Performance of regenerating problems with child providers

Documentation/DataLoadEngine/RegexRedactions.md

@@ -0,0 +1,44 @@
+# Regex Redactions
+
+RDMP has the ability to redact incoming and existing catalogue data.
+It uses user-defined regex to do this.
+This functionality may be useful if you know about problematic data that you wish to clean before it reaches RDMP
+
+## Setup & Configuration
+To begin, a redaction configuration is required.
+This can be added in the "Configurations" section of RDMP.
+Each configuration has:
+* A Name
+* A Description
+* A Regex string to match
+* A Replacement value
+
+N.B. The replacement value cannot be longer than the redacted string i.e. "string" => "myRedactedString" will not work.
+
+Once the configuration has been saved, it can be used to redact data.
+
+## Usage in Data Loads
+Within the Data Loads mutilators, there is a mutilator named "Regex Redaction Mutilator" that can be used in the RAW or STAGING areas of the load.
+This mutilator will redact based on your selected configuration.
+The configuration for this mutilator requires:
+* A predefined regex redaction configuration (see above)
+* Either a regex to select which columns to redact, or a selection of known columns
+
+N.B. Primary Key columns will not be redacted and this functionality is intended for string fields.
+
+Once the configuration has been set, it will redact the desired columns during a data load.
+
+During a run, the mutilator will replace any regex matches with the desired string and store the now known redaction in RDMP.
+To view the redactions made, see the section below.
+
+
+## Usage within RDMP
+You can view and restore existing redactions, along with adding new redactions within RDMP.
+For a Catalogue, right-click and open the "Catalogue Items" submenu. 
+From here, select the "Regex Redactions" option.
+This will open an interface displaying existing redactions.
+From here you can restore existing redactions either individually or in bulk.
+You can also apply regex redactions to existing data by selecting a redaction configuration and which columns to redact.
+
+
+[Catalogue](../CodeTutorials/Glossary.md#Catalogue)

Documentation/DataLoadEngine/RemoteAttachers.md

@@ -50,6 +50,7 @@ The full configuration options are
 | Culture                                      | Optionally specify a custom date format                                                                                                                                                                                                                                                                                                                     |
 | Explicit Date Time Format                    | Optionally specify a specific datetime format    
 | Selected Columns                             | Optionally select which columns you wish to pull from the remote server (defaults to "*" )
+| RawTableDateColumn                           | Optionally give the date column in RAW a different name than in the select statement for delta loads. This may be due to awkward joins in a custom select query 
 
 ## Configuring the Remote Database Attacher
 The Remote Database Attacher has a number of configuration options the required fields are:

HIC.DataManagementPlatform.sln

@@ -10,8 +10,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		CHANGELOG.md = CHANGELOG.md
 		deadlinksconfig.json = deadlinksconfig.json
 		Directory.Build.props = Directory.Build.props
-		.github\workflows\links.yml = .github\workflows\links.yml
 		Directory.Packages.props = Directory.Packages.props
+		.github\workflows\links.yml = .github\workflows\links.yml
 		NoteForNewDevelopers.md = NoteForNewDevelopers.md
 		Documentation\CodeTutorials\Packages.md = Documentation\CodeTutorials\Packages.md
 		rdmp-client.xml = rdmp-client.xml
@@ -55,6 +55,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Documentation", "Documentat
 		Documentation\CodeTutorials\Packages.md = Documentation\CodeTutorials\Packages.md
 		Documentation\CodeTutorials\PluginWriting.md = Documentation\CodeTutorials\PluginWriting.md
 		Documentation\CodeTutorials\RdmpCommandLine.md = Documentation\CodeTutorials\RdmpCommandLine.md
+		Documentation\DataLoadEngine\RegexRedactions.md = Documentation\DataLoadEngine\RegexRedactions.md
 		Documentation\CodeTutorials\Tests.md = Documentation\CodeTutorials\Tests.md
 		Documentation\CodeTutorials\UserInterfaceOverview.md = Documentation\CodeTutorials\UserInterfaceOverview.md
 		Documentation\CodeTutorials\UserManual.md = Documentation\CodeTutorials\UserManual.md
@@ -315,4 +316,4 @@ Global
 	GlobalSection(Performance) = preSolution
 		HasPerformanceSessions = true
 	EndGlobalSection
-EndGlobal
+EndGlobal