more resiliancy (#4395)

Author: mr-c

src/toil/fileStores/nonCachingFileStore.py

@@ -126,7 +126,11 @@ def open(self, job: Job) -> Generator[None, None, None]:
             os.chdir(startingDir)
             # Finally delete the job from the worker
             self.check_for_state_corruption()
-            os.remove(self.jobStateFile)
+            try:
+                os.remove(self.jobStateFile)
+            except FileNotFoundError:
+                logger.exception('Job state file %s has gone missing unexpectedly; some cleanup for failed jobs may be getting skipped!', self.jobStateFile)
+                pass
 
     def writeGlobalFile(self, localFileName: str, cleanup: bool=False) -> FileID:
         absLocalFileName = self._resolveAbsoluteLocalPath(localFileName)

src/toil/statsAndLogging.py

@@ -148,6 +148,8 @@ def callback(fileHandle: Union[IO[bytes], IO[str]]) -> None:
             if not isinstance(statsStr, str):
                 statsStr = statsStr.decode()
             stats = json.loads(statsStr, object_hook=Expando)
+            if not stats:
+                return
             try:
                 logs = stats.workers.logsToMaster
             except AttributeError: