Merge pull request #103 from SMI/feature/filesizes

Feature/filesizes
SMI · Feb 6, 2020 · e225d10 · e225d10
2 parents 3261594 + 4eb728c
commit e225d10
Show file tree

Hide file tree

Showing 13 changed files with 57 additions and 23 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Added
 
+- Added (optional) DicomFileSize property to ETL pipeline.  Add to template(s) with:
+```yaml
+  - ColumnName: DicomFileSize
+    AllowNulls: true
+    Type:
+      CSharpType: System.Int64
+```
+
 - Added new microservice IsIdentifiable which scans for personally identifiable information (in databases and dicom files)
 - Added support for custom rules in IsIdentifiable (entered in `Rules.yaml`)
   - Rules are applied in the order they appear in this file

diff --git a/src/common/Smi.Common.MongoDb/MongoDocumentHeaders.cs b/src/common/Smi.Common.MongoDb/MongoDocumentHeaders.cs
@@ -19,6 +19,7 @@ public static BsonDocument ImageDocumentHeader(DicomFileMessage message, IMessag
             return new BsonDocument
             {
                 { "DicomFilePath",               message.DicomFilePath },
+                { "DicomFileSize",               message.DicomFileSize },
                 { "NationalPACSAccessionNumber", message.NationalPACSAccessionNumber },
                 { "MessageHeader", new BsonDocument
                     {

diff --git a/src/common/Smi.Common/Messages/DicomFileMessage.cs b/src/common/Smi.Common/Messages/DicomFileMessage.cs
@@ -25,6 +25,8 @@ public sealed class DicomFileMessage : IComparable, IFileReferenceMessage
         [JsonProperty(Required = Required.Always)]
         public string DicomFilePath { get; set; }
 
+        public long DicomFileSize { get; set; } = -1;
+
         /// <summary>
         /// Dicom tag (0020,000D).
         /// </summary>

diff --git a/src/microservices/Microservices.DicomRelationalMapper/DicomRelationalMapper.cd b/src/microservices/Microservices.DicomRelationalMapper/DicomRelationalMapper.cd
@@ -55,6 +55,7 @@
     <ShowAsAssociation>
       <Property Name="Consumer" />
     </ShowAsAssociation>
+    <Lollipop Position="0.2" />
   </Class>
   <Class Name="Microservices.DicomRelationalMapper.Execution.NLogThrowerDataLoadEventListener" Collapsed="true" BaseTypeListCollapsed="true">
     <Position X="5.5" Y="10.25" Width="2.75" />
@@ -74,24 +75,26 @@
   <Class Name="Microservices.DicomRelationalMapper.Messaging.DicomRelationalMapperQueueConsumer">
     <Position X="3.75" Y="4.5" Width="3" />
     <TypeIdentifier>
-      <HashCode>gIAAAEDiAECBACBAAAABAAAAQAAAAEAAcAEGAACAAAA=</HashCode>
+      <HashCode>gIAAAEDiAGCBICBAAAABAAAAQAAAAEABcAEGAACAAAA=</HashCode>
       <FileName>Messaging\DicomRelationalMapperQueueConsumer.cs</FileName>
     </TypeIdentifier>
+    <Lollipop Position="0.2" />
   </Class>
-  <Class Name="Microservices.Common.Options.GlobalOptions" Collapsed="true">
+  <Class Name="Smi.Common.Options.GlobalOptions" Collapsed="true">
     <Position X="0.5" Y="0.5" Width="1.5" />
-    <AssociationLine Name="RDMPOptions" Type="Microservices.Common.Options.RDMPOptions" FixedFromPoint="true" FixedToPoint="true">
+    <AssociationLine Name="RDMPOptions" Type="Smi.Common.Options.RDMPOptions" FixedFromPoint="true" FixedToPoint="true">
       <Path>
-        <Point X="1.75" Y="1.062" />
-        <Point X="1.75" Y="1.531" />
+        <Point X="2" Y="0.938" />
+        <Point X="2.375" Y="0.938" />
+        <Point X="2.375" Y="1.531" />
         <Point X="6" Y="1.531" />
       </Path>
     </AssociationLine>
-    <AssociationLine Name="DicomRelationalMapperOptions" Type="Microservices.Common.Options.DicomRelationalMapperOptions" FixedFromPoint="true" FixedToPoint="true">
+    <AssociationLine Name="DicomRelationalMapperOptions" Type="Smi.Common.Options.DicomRelationalMapperOptions" ManuallyRouted="true" FixedFromPoint="true" FixedToPoint="true">
       <Path>
-        <Point X="1.312" Y="1.062" />
-        <Point X="1.312" Y="3" />
-        <Point X="6" Y="3" />
+        <Point X="1.5" Y="1.062" />
+        <Point X="1.5" Y="3" />
+        <Point X="8.25" Y="3" />
       </Path>
     </AssociationLine>
     <TypeIdentifier />
@@ -101,16 +104,16 @@
       <Property Name="DicomRelationalMapperOptions" />
     </ShowAsAssociation>
   </Class>
-  <Class Name="Microservices.Common.Options.DicomRelationalMapperOptions">
-    <Position X="6" Y="2" Width="2.75" />
+  <Class Name="Smi.Common.Options.RabbitOptions" Collapsed="true">
+    <Position X="6" Y="0.5" Width="1.5" />
     <TypeIdentifier />
   </Class>
-  <Class Name="Microservices.Common.Options.RDMPOptions" Collapsed="true">
+  <Class Name="Smi.Common.Options.RDMPOptions" Collapsed="true">
     <Position X="6" Y="1.25" Width="1.5" />
     <TypeIdentifier />
   </Class>
-  <Class Name="Microservices.Common.Options.RabbitOptions" Collapsed="true">
-    <Position X="6" Y="0.5" Width="1.5" />
+  <Class Name="Smi.Common.Options.DicomRelationalMapperOptions">
+    <Position X="8.25" Y="0.5" Width="2.75" />
     <TypeIdentifier />
   </Class>
   <Interface Name="Microservices.DicomRelationalMapper.Execution.Namers.ICreateAndDestroyStagingDuringLoads">

diff --git a/...es/Microservices.DicomRelationalMapper/Execution/DicomFileMessageToDatasetListProvider.cs b/...es/Microservices.DicomRelationalMapper/Execution/DicomFileMessageToDatasetListProvider.cs
@@ -32,6 +32,7 @@ public DicomDataset GetNextDatasetToProcess(out string filename, out Dictionary<
             filename = toReturn.DicomFileMessage.DicomFilePath;
 
             otherValuesToStoreInRow.Add("MessageGuid", _messages[_progress].Header.MessageGuid.ToString());
+            otherValuesToStoreInRow.Add("DicomFileSize",toReturn.DicomFileMessage.DicomFileSize.ToString()); //TN: It won't be a string when it hits the database but the API supports only string/string for this out Dictionary
 
             _progress++;
 

diff --git a/src/microservices/Microservices.DicomReprocessor/Execution/Processors/DicomFileProcessor.cs b/src/microservices/Microservices.DicomReprocessor/Execution/Processors/DicomFileProcessor.cs
@@ -9,6 +9,7 @@
 using Smi.Common.Options;
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading;
 
 
@@ -73,7 +74,8 @@ public void ProcessDocument(BsonDocument document)
             var message = new DicomFileMessage
             {
                 NationalPACSAccessionNumber = (string)headerDoc["NationalPACSAccessionNumber"],
-                DicomFilePath = (string)headerDoc["DicomFilePath"]
+                DicomFilePath = (string)headerDoc["DicomFilePath"],
+                DicomFileSize = headerDoc.Contains("DicomFileSize") ? (long)headerDoc["DicomFileSize"] : -1
             };
 
             try

diff --git a/src/microservices/Microservices.DicomTagReader/Execution/ParallelTagReader.cs b/src/microservices/Microservices.DicomTagReader/Execution/ParallelTagReader.cs
@@ -1,6 +1,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.IO.Abstractions;
 using System.Threading;
 using System.Threading.Tasks;
@@ -27,7 +28,7 @@ public ParallelTagReader(DicomTagReaderOptions options, FileSystemOptions fileSy
             Logger.Info($"Using MaxDegreeOfParallelism={_parallelOptions.MaxDegreeOfParallelism} for parallel IO operations");
         }
 
-        protected override List<DicomFileMessage> ReadTagsImpl(IEnumerable<string> dicomFilePaths,
+        protected override List<DicomFileMessage> ReadTagsImpl(IEnumerable<FileInfo> dicomFilePaths,
             AccessionDirectoryMessage accMessage)
         {
             var fileMessages = new List<DicomFileMessage>();

diff --git a/src/microservices/Microservices.DicomTagReader/Execution/SerialTagReader.cs b/src/microservices/Microservices.DicomTagReader/Execution/SerialTagReader.cs
@@ -1,6 +1,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.IO.Abstractions;
 using Smi.Common.Messages;
 using Smi.Common.Messaging;
@@ -15,11 +16,11 @@ public SerialTagReader(DicomTagReaderOptions options, FileSystemOptions fileSyst
             IProducerModel seriesMessageProducerModel, IProducerModel fileMessageProducerModel, IFileSystem fs)
             : base(options, fileSystemOptions, seriesMessageProducerModel, fileMessageProducerModel, fs) { }
 
-        protected override List<DicomFileMessage> ReadTagsImpl(IEnumerable<string> dicomFilePaths, AccessionDirectoryMessage accMessage)
+        protected override List<DicomFileMessage> ReadTagsImpl(IEnumerable<FileInfo> dicomFilePaths, AccessionDirectoryMessage accMessage)
         {
             var fileMessages = new List<DicomFileMessage>();
 
-            foreach (string dicomFilePath in dicomFilePaths)
+            foreach (FileInfo dicomFilePath in dicomFilePaths)
             {
                 Logger.Trace("TagReader: Processing " + dicomFilePath);
 

diff --git a/src/microservices/Microservices.DicomTagReader/Execution/TagReaderBase.cs b/src/microservices/Microservices.DicomTagReader/Execution/TagReaderBase.cs
@@ -2,6 +2,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.IO;
 using System.IO.Abstractions;
 using System.Linq;
 using System.Text;
@@ -97,7 +98,7 @@ public void ReadTags(IMessageHeader header, AccessionDirectoryMessage message)
 
             long beginRead = _stopwatch.ElapsedTicks;
 
-            List<DicomFileMessage> fileMessages = ReadTagsImpl(dicomFilePaths, message);
+            List<DicomFileMessage> fileMessages = ReadTagsImpl(dicomFilePaths.Select(p=>new FileInfo(p)), message);
 
             _swTotals[1] += (_stopwatch.ElapsedTicks - beginRead) / dicomFilePaths.Length;
 
@@ -169,23 +170,23 @@ public void ReadTags(IMessageHeader header, AccessionDirectoryMessage message)
                 LogRates();
         }
 
-        protected abstract List<DicomFileMessage> ReadTagsImpl(IEnumerable<string> dicomFilePaths,
+        protected abstract List<DicomFileMessage> ReadTagsImpl(IEnumerable<FileInfo> dicomFilePaths,
             AccessionDirectoryMessage accMessage);
 
         /// <summary>
         /// Builds a <see cref="DicomFileMessage"/> from a single dicom file
         /// </summary>
         /// <param name="dicomFilePath"></param>
         /// <returns></returns>
-        protected static DicomFileMessage ReadTagsFromFile(string dicomFilePath)
+        protected static DicomFileMessage ReadTagsFromFile(FileInfo dicomFilePath)
         {
             DicomDataset ds;
             var IDs = new string[3];
 
             try
             {
                 //TODO(Ruairidh 04/07) Check if we can mock this out now
-                ds = DicomFile.Open(dicomFilePath, _fileReadOption).Dataset;
+                ds = DicomFile.Open(dicomFilePath.FullName, _fileReadOption).Dataset;
 
                 // Pre-fetch these to ensure they exist before we go further
                 IDs[0] = ds.GetValue<string>(DicomTag.StudyInstanceUID, 0);
@@ -222,7 +223,8 @@ protected static DicomFileMessage ReadTagsFromFile(string dicomFilePath)
                 SeriesInstanceUID = IDs[1],
                 SOPInstanceUID = IDs[2],
 
-                DicomDataset = serializedDataset
+                DicomDataset = serializedDataset,
+                DicomFileSize = dicomFilePath.Length
             };
         }
 

diff --git a/tests/common/Smi.Common.MongoDb.Tests/MongoDocumentHeadersTests.cs b/tests/common/Smi.Common.MongoDb.Tests/MongoDocumentHeadersTests.cs
@@ -35,6 +35,7 @@ public void ImageDocumentHeader_HasCorrectHeaders()
             var expected = new BsonDocument
             {
                 { "DicomFilePath",               msg.DicomFilePath },
+                { "DicomFileSize",               msg.DicomFileSize },
                 { "NationalPACSAccessionNumber", msg.NationalPACSAccessionNumber },
                 { "MessageHeader", new BsonDocument
                 {

diff --git a/tests/microservices/Microservices.DicomRelationalMapper.Tests/DLEBenchmarkingTests/CT.it b/tests/microservices/Microservices.DicomRelationalMapper.Tests/DLEBenchmarkingTests/CT.it
@@ -120,3 +120,7 @@ Tables:
     AllowNulls: true
   - ColumnName: ScanOptions
     AllowNulls: true
+  - ColumnName: DicomFileSize
+    AllowNulls: true
+    Type:
+      CSharpType: System.Int64
diff --git a/...icroservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperTestHelper.cs b/...icroservices/Microservices.DicomRelationalMapper.Tests/DicomRelationalMapperTestHelper.cs
@@ -115,6 +115,7 @@ public DicomFileMessage GetDicomFileMessage(string fileSystemRoot, FileInfo fi)
             toReturn.StudyInstanceUID = "999";
             toReturn.SeriesInstanceUID = "999";
             toReturn.SOPInstanceUID = "999";
+            toReturn.DicomFileSize = fi.Length;
 
             var ds = DicomFile.Open(fi.FullName).Dataset;
             ds.Remove(DicomTag.PixelData);
@@ -262,6 +263,10 @@ public DicomFileMessage GetDicomFileMessage(DicomDataset ds, string fileSystemRo
     AllowNulls: true
   - ColumnName: ScanOptions
     AllowNulls: true
+  - ColumnName: DicomFileSize
+    AllowNulls: true
+    Type:
+      CSharpType: System.Int64
 ";
     }
 }
diff --git a/...s/microservices/Microservices.DicomRelationalMapper.Tests/MicroservicesIntegrationTest.cs b/...s/microservices/Microservices.DicomRelationalMapper.Tests/MicroservicesIntegrationTest.cs
@@ -493,6 +493,9 @@ private void RunTest(DirectoryInfo dir, int numberOfExpectedRows, Action<FileSys
                     //make sure that the substitution identifier (that replaces old the PatientId) is the correct substitution (FISHFISH)/
                     Assert.AreEqual("FISHFISH", _helper.StudyTable.GetDataTable().Rows.OfType<DataRow>().First()["PatientId"]);
 
+                    //The file size in the final table should be more than 0
+                    Assert.Greater((long)_helper.ImageTable.GetDataTable().Rows.OfType<DataRow>().First()["DicomFileSize"],0);
+
                     dicomTagReaderHost.Stop("TestIsFinished");
 
                     mongoDbPopulatorHost.Stop("TestIsFinished");