Move bad-files under each volume directory

The bad-files directory, currently sitting directly under the NGAS root
directory and not within any of the volume directories, contains files
that for one reason or another could not be archived automatically
(server shutdown, client disconnected, etc). When files are moved into
the bad-files directory, their original location is always the staging/
directory in the target volume where the final archiving takes place.
Because volumes are usually mapped to different filesystems, this often
results on slow, expensive data copies across filesystems instead of
fast, simpler renames within the single filesystem. Moreover, files
under bad-files are never automatically removed or picked out in any
way, and therefore they simply accumulate until they are dealt with
(most commonly removed).

This commit moves the bad-files directory to exist within each of the
volumes rather than directly under the NGAS root. This should speed up
greatly the movement of files into such directories, while keeping the
information of the disk they originated from.

This issue was reported in #51.

Signed-off-by: Rodrigo Tobar <[email protected]>

Author: rtobar

doc/changelog.rst

@@ -96,6 +96,14 @@ Changelog
   when registration of a file
   with a MIME type with no configured plug-in
   is requested.
+* The ``bad-files`` directory
+  now exists on each volume
+  rather that there being a single one
+  outside of any volume directory.
+  This allows for faster movement of files
+  into the ``bad-files`` directory,
+  a more consistent directory structure,
+  and better traceability of files that end up in ``bad-files``.
 
 .. rubric:: 11.0.2
 

src/ngamsCore/ngamsLib/ngamsHighLevelLib.py

@@ -524,14 +524,14 @@ def getNewFileVersion(dbConObj,
         return (latestFileVersion + 1)
 
 
-def moveFile2BadDir(ngamsCfgObj, srcFilename):
+def moveFile2BadDir(disk, srcFilename):
     """
     Move a file to the Bad File Directory on the destination disk.
 
     Returns:         Name of filename in Bad Files Area (string).
     """
 
-    badFilesDir = os.path.join(ngamsCfgObj.getRootDirectory(), NGAMS_BAD_FILES_DIR)
+    badFilesDir = os.path.join(disk.getMountPoint(), NGAMS_BAD_FILES_DIR)
     checkCreatePath(badFilesDir)
 
     trgFilename = tempfile.mktemp(suffix="-" + os.path.basename(srcFilename),

src/ngamsServer/ngamsServer/ngamsArchiveUtils.py

@@ -769,7 +769,7 @@ def archiveFromFile(srvObj,
                   ". Attempt failed with following error", filename)
             logger.warning("Moving local file: " +\
                    filename + " to Bad Files Directory -- cannot be handled.")
-            ngamsHighLevelLib.moveFile2BadDir(srvObj.getCfg(), filename)
+            ngamsHighLevelLib.moveFile2BadDir(trgDiskInfo, filename)
             # Remove pickle file if available.
             pickleObjFile = filename + "." + NGAMS_PICKLE_FILE_EXT
             if (os.path.exists(pickleObjFile)):

src/ngamsServer/ngamsServer/ngamsSrvUtils.py

@@ -450,18 +450,13 @@ def checkStagingAreas(srvObj):
                getDiskInfoForMountedDisks(srvObj.getDb(), srvObj.getHostId(),
                                           srvObj.getCfg().\
                                           getRootDirectory())
-    # Generate first a dictionary with all files in the staging areas.
-    stagingFileDic = {}
     for disk in diskList:
         stagingArea = disk.getStagingArea()
-        if (stagingArea != ""):
-            fileList = glob.glob(stagingArea + "/*")
-            fileList += glob.glob(stagingArea + "/.NGAMS*")
-            for filename in fileList: stagingFileDic[filename] = 1
-    # Go through all files in the staging file dictionary and move them to
-    # the Bad Files Area.
-    for filename in stagingFileDic.keys():
-        ngamsHighLevelLib.moveFile2BadDir(srvObj.getCfg(), filename)
+        if stagingArea:
+            files_to_move = glob.glob(stagingArea + "/*")
+            files_to_move += glob.glob(stagingArea + "/.NGAMS*")
+            for f in files_to_move:
+                ngamsHighLevelLib.moveFile2BadDir(disk, f)
 
 
 def genIntAuthHdr(srvObj):

test/commands/test_archive.py

@@ -966,9 +966,9 @@ def test_ArchiveRobustness_01_01(self):
             fo.close()
 
         self.restart_last_server()
-        badDirPat = self.ngas_path("bad-files/BAD-FILE-*-%s.fits")
+        badDirPat = self.ngas_path("%s/bad-files/BAD-FILE-*-%s.fits")
         for diskName in diskList:
-            badFile = badDirPat % diskName
+            badFile = badDirPat % (diskName, diskName)
             pollForFile(badFile, 1)
             pollForFile("%s.%s" % (badFile, NGAMS_PICKLE_FILE_EXT), 1)
 
@@ -1011,7 +1011,7 @@ def before_restart():
             self.assertGreater(len(glob.glob(req_fname)), 0)
 
         self.restart_last_server(before_restart=before_restart, force=True)
-        reqPropBadFile = self.ngas_path("bad-files/BAD-FILE-*-SmallFile.fits.pickle", port=8888)
+        reqPropBadFile = self.ngas_path("FitsStorage1-Main-1/bad-files/BAD-FILE-*-SmallFile.fits.pickle", port=8888)
         pollForFile(reqPropBadFile, 1)