Merge pull request #635 from jkloetzke/fix-delete-extract

Fix extraction of archives if workspace was deleted

Author: jkloetzke

pym/bob/archive.py

@@ -30,7 +30,6 @@
 import concurrent.futures
 import concurrent.futures.process
 import gzip
-import http.client
 import io
 import os
 import os.path

pym/bob/builder.py

@@ -718,7 +718,7 @@ def linkTo(dest, linkName):
                                         "{:02}-{}".format(i, a.getPackage().getName())))
                 i += 1
 
-    async def _runShell(self, step, scriptName, logger, cleanWorkspace=None,
+    async def _runShell(self, step, scriptName, logger, workspaceCreated, cleanWorkspace=None,
                         mode=InvocationMode.CALL):
         workspacePath = step.getWorkspacePath()
         if not os.path.isdir(workspacePath): os.makedirs(workspacePath)
@@ -771,7 +771,7 @@ async def _runShell(self, step, scriptName, logger, cleanWorkspace=None,
             executor=self.__executor)
         if step.jobServer() and self.__jobServer:
             invoker.setMakeParameters(self.__jobServer.getMakeFd(), self.__jobs)
-        ret = await invoker.executeStep(mode, cleanWorkspace)
+        ret = await invoker.executeStep(mode, workspaceCreated, cleanWorkspace)
         if not self.__bufferedStdIO: ttyReinit() # work around MSYS2 messing up the console
         if ret == -int(signal.SIGINT):
             raise BuildError("User aborted while running {}".format(absRunFile),
@@ -1110,7 +1110,7 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
                 if checkoutBuildOnlyStateCompatible(checkoutState, oldCheckoutState):
                     with stepExec(checkoutStep, "UPDATE",
                                   "{} {}".format(prettySrcPath, overridesString)) as a:
-                        await self._runShell(checkoutStep, "checkout", a, mode=InvocationMode.UPDATE)
+                        await self._runShell(checkoutStep, "checkout", a, created, mode=InvocationMode.UPDATE)
                     newCheckoutState = oldCheckoutState.copy()
                     newCheckoutState[CHECKOUT_STATE_BUILD_ONLY] = checkoutState[CHECKOUT_STATE_BUILD_ONLY]
                     BobState().setDirectoryState(prettySrcPath, newCheckoutState)
@@ -1235,7 +1235,7 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
 
                 with stepExec(checkoutStep, "CHECKOUT",
                               "{} ({}) {}".format(prettySrcPath, checkoutReason, overridesString)) as a:
-                    await self._runShell(checkoutStep, "checkout", a)
+                    await self._runShell(checkoutStep, "checkout", a, created)
                 self.__statistic.checkouts += 1
                 checkoutExecuted = True
                 currentResultHash.invalidate() # force recalculation
@@ -1302,6 +1302,7 @@ async def _cookBuildStep(self, buildStep, depth, buildBuildId):
                 stepMessage(buildStep, "PRUNE", "{} (recipe changed)".format(prettyBuildPath),
                     WARNING)
                 emptyDirectory(prettyBuildPath)
+                created = True
             # invalidate build step
             BobState().resetWorkspaceState(prettyBuildPath, buildDigest)
 
@@ -1327,7 +1328,7 @@ async def _cookBuildStep(self, buildStep, depth, buildBuildId):
                 # build it
                 with stepExec(buildStep, "BUILD", prettyBuildPath, INFO) as buildAction:
                     visibleAction = fullAction if fullAction.visible else buildAction
-                    await self._runShell(buildStep, "build", visibleAction, self.__cleanBuild)
+                    await self._runShell(buildStep, "build", visibleAction, created, self.__cleanBuild)
                 buildHash = hashWorkspace(buildStep)
                 await self._generateAudit(buildStep, depth, buildHash, buildBuildId)
             BobState().setResultHash(prettyBuildPath, buildHash)
@@ -1577,7 +1578,7 @@ async def _cookPackageStep(self, packageStep, depth, packageBuildId):
                 BobState().setResultHash(prettyPackagePath, datetime.datetime.now())
                 with stepExec(packageStep, "PACKAGE", prettyPackagePath, INFO) as packageAction:
                     visibleAction = fullAction if fullAction.visible else packageAction
-                    await self._runShell(packageStep, "package", visibleAction)
+                    await self._runShell(packageStep, "package", visibleAction, True)
                 packageHash = hashWorkspace(packageStep)
                 packageDigest = await self.__getIncrementalVariantId(packageStep)
                 workspaceChanged = True

pym/bob/cmds/invoke.py

@@ -50,14 +50,14 @@ def doInvoke(argv, bobRoot):
             invoker = Invoker(spec, args.preserve_env, True, True, True, False,
                               False, executor=executor)
             ret = loop.run_until_complete(invoker.executeStep(InvocationMode.SHELL,
-                args.clean, args.keep_sandbox))
+                False, args.clean, args.keep_sandbox))
         elif args.mode in ('run', 'update'):
             invoker = Invoker(spec, args.preserve_env, args.no_logfiles,
                 verbosity >= 2, verbosity >= 1, verbosity >= 3, False,
                 executor=executor)
             ret = loop.run_until_complete(invoker.executeStep(
                 InvocationMode.CALL if args.mode == 'run' else InvocationMode.UPDATE,
-                args.clean, args.keep_sandbox))
+                False, args.clean, args.keep_sandbox))
         elif args.mode == 'fingerprint':
             invoker = Invoker(spec, args.preserve_env, True, True, True,
                               verbosity >= 3, False, executor=executor)

pym/bob/invoker.py

@@ -334,7 +334,7 @@ def __getSlimSandboxCmds(self, tmpDir):
 
         return cmdArgs
 
-    async def executeStep(self, mode, clean=False, keepSandbox=False):
+    async def executeStep(self, mode, workspaceCreated, clean=False, keepSandbox=False):
         # make permissions predictable
         os.umask(0o022)
 
@@ -352,8 +352,10 @@ async def executeStep(self, mode, clean=False, keepSandbox=False):
             clean = self.__spec.clean if self.__spec.clean is not None else clean
             if not os.path.isdir(self.__spec.workspaceWorkspacePath):
                 os.makedirs(self.__spec.workspaceWorkspacePath, exist_ok=True)
+                workspaceCreated = True
             elif clean and mode != InvocationMode.SHELL:
                 emptyDirectory(self.__spec.workspaceWorkspacePath)
+                workspaceCreated = True
 
             if len(self.__makeFds) == 2:
                 makeFlags = self.__spec.env.get("MAKEFLAGS")
@@ -426,7 +428,7 @@ async def executeStep(self, mode, clean=False, keepSandbox=False):
                     if mode == InvocationMode.UPDATE and not scm.isLocal():
                         continue # Skip non-local SCMs on update-only
                     try:
-                        await scm.invoke(self)
+                        await scm.invoke(self, workspaceCreated)
                     except CmdFailedError as e:
                         self.error(scm.getSource(), "failed")
                         self.error(e.what)

pym/bob/layers.py

@@ -155,7 +155,7 @@ async def __checkoutTask(self, verbose, attic):
             os.makedirs(self.__layerDir)
             self.__created = True
 
-        await self.__scm.invoke(invoker)
+        await self.__scm.invoke(invoker, self.__created)
         log("CHECKOUT: Layer " +
                 "'{}' .. ok".format(self.getName()), EXECUTED, NORMAL)
         BobState().setLayerState(self.__layerDir, newState)

pym/bob/scm/cvs.py

@@ -61,7 +61,7 @@ def getProperties(self, isJenkins, pretty=False):
 
         return ret
 
-    async def invoke(self, invoker):
+    async def invoke(self, invoker, workspaceCreated):
         # If given a ":ssh:" cvsroot, translate that to CVS_RSH using ssh, and ":ext:"
         # (some versions of CVS do that internally)
         m = re.match('^:ssh:(.*)', self.__cvsroot)

pym/bob/scm/git.py

@@ -206,7 +206,7 @@ def _getGitConfigOptions(self):
             config += [ "-c", "protocol.file.allow=always" ]
         return config
 
-    async def invoke(self, invoker, switch=False):
+    async def invoke(self, invoker, workspaceCreated, switch=False):
         alternatesFile = invoker.joinPath(self.__dir, ".git/objects/info/alternates")
 
         # make sure the git directory exists
@@ -709,7 +709,7 @@ async def switch(self, invoker, oldScm):
 
         # Try to checkout new state in old workspace. If something fails the
         # old attic logic will take over.
-        await self.invoke(invoker, True)
+        await self.invoke(invoker, False, True)
         return True
 
     def asDigestScript(self):

pym/bob/scm/imp.py

@@ -166,7 +166,7 @@ def getProperties(self, isJenkins, pretty=False):
 
         return ret
 
-    async def invoke(self, invoker):
+    async def invoke(self, invoker, workspaceCreated):
         dest = invoker.joinPath(self.__dir)
         os.makedirs(dest, exist_ok=True)
         if self.__prune: emptyDirectory(dest)

pym/bob/scm/svn.py

@@ -72,7 +72,7 @@ def getProperties(self, isJenkins, pretty=False):
 
         return ret
 
-    async def invoke(self, invoker):
+    async def invoke(self, invoker, workspaceCreated):
         options = [ "--non-interactive" ]
         if not self.__sslVerify:
             options += [ "--trust-server-cert-failures=unknown-ca,cn-mismatch,expired,not-yet-valid,other" ]

pym/bob/scm/url.py

@@ -168,11 +168,11 @@ def __init__(self, dir, file, strip, separateDownload):
         self.strip = strip
         self.separateDownload = separateDownload
 
-    async def _extract(self, cmds, invoker, stdout=None):
+    async def _extract(self, cmds, invoker, dirCreated, stdout=None):
         destination = self.getCompressedFilePath(invoker)
         (destDir, destFile) = os.path.split(destination)
         canary = os.path.join(destDir, "."+destFile+".extracted")
-        if isYounger(destination, canary):
+        if dirCreated or isYounger(destination, canary):
             for cmd in cmds:
                 if shutil.which(cmd[0]) is None: continue
                 await invoker.checkCommand(cmd, cwd=self.dir, stdout=stdout)
@@ -189,7 +189,7 @@ def getCompressedFilePath(self, invoker):
         return os.path.abspath(invoker.joinPath(downloadFolder, self.dir, self.file)) \
 
     @abstractmethod
-    async def extract(self, invoker, destination, cwd):
+    async def extract(self, invoker, dirCreated):
         return False
 
 # Use the Python tar/zip extraction only on Windows. They are slower and in
@@ -198,7 +198,7 @@ async def extract(self, invoker, destination, cwd):
 class TarExtractor(Extractor):
     SUPPORT_STRIP = True
 
-    async def extract(self, invoker):
+    async def extract(self, invoker, dirCreated):
         cmds = []
         compressedFilePath = self.getCompressedFilePath(invoker)
         if isWin32 and self.strip == 0:
@@ -210,25 +210,25 @@ async def extract(self, invoker):
             cmd.append("--strip-components={}".format(self.strip))
         cmds.append(cmd)
 
-        await self._extract(cmds, invoker)
+        await self._extract(cmds, invoker, dirCreated)
 
 
 class ZipExtractor(Extractor):
 
-    async def extract(self, invoker):
+    async def extract(self, invoker, dirCreated):
         cmds = []
         compressedFilePath = self.getCompressedFilePath(invoker)
         if isWin32:
             cmds.append(["python", "-m", "zipfile",
                    "-e", compressedFilePath, "."])
 
         cmds.append(["unzip", "-o", compressedFilePath])
-        await self._extract(cmds, invoker)
+        await self._extract(cmds, invoker, dirCreated)
 
 
 class SingleFileExtractor(Extractor):
 
-    async def extract(self, invoker):
+    async def extract(self, invoker, dirCreated):
         # The gunzip and unxz tools extracts the file at the location of the
         # input file. In case the separateDownload policy is active, the
         # destination might be in a separete folder.
@@ -248,7 +248,7 @@ async def extract(self, invoker):
 
         with open(dst, 'wb') as f:
             cmd = [self.CMD, "-c", src]
-            await self._extract([cmd], invoker, f)
+            await self._extract([cmd], invoker, dirCreated, f)
 
         shutil.copystat(src, dst)
 
@@ -281,9 +281,9 @@ class XZExtractor(SingleFileExtractor):
 
 class SevenZipExtractor(Extractor):
 
-    async def extract(self, invoker):
+    async def extract(self, invoker, dirCreated):
         cmds = [["7z", "x", "-y", self.getCompressedFilePath(invoker)]]
-        await self._extract(cmds, invoker)
+        await self._extract(cmds, invoker, dirCreated)
 
 
 class UrlScm(Scm):
@@ -685,8 +685,10 @@ async def switch(self, invoker, oldScm):
         # does not match.
         return True
 
-    async def invoke(self, invoker):
-        os.makedirs(invoker.joinPath(self.__dir), exist_ok=True)
+    async def invoke(self, invoker, workspaceCreated):
+        if not os.path.isdir(invoker.joinPath(self.__dir)):
+            os.makedirs(invoker.joinPath(self.__dir), exist_ok=True)
+            workspaceCreated = True
         workspaceFile = os.path.join(self.__dir, self.__fn)
         extractor = self.__getExtractor()
 
@@ -747,7 +749,7 @@ async def invoke(self, invoker):
 
         # Run optional extractors
         if extractor is not None:
-            await extractor.extract(invoker)
+            await extractor.extract(invoker, workspaceCreated)
 
     def asDigestScript(self):
         """Return forward compatible stable string describing this url.