Issue #2065 - Part 2: Expand pattern when track file is created rathe…

…r than read

This fixes build bustage on Windows when using `mach build faster`.

Based on https://bugzilla.mozilla.org/show_bug.cgi?id=1416465

CLOBBER

@@ -22,4 +22,4 @@
 # changes to stick? As of bug 928195, this shouldn't be necessary! Please
 # don't change CLOBBER for WebIDL changes any more.
 
-Issue #1971 - Update to ICU 63 requires clobber
+Clobber for removing invalid track files with wildcards

python/mozbuild/mozbuild/action/process_install_manifest.py

@@ -70,7 +70,9 @@ def process_manifest(destdir, paths, track=None,
         remove_empty_directories=remove_empty_directories)
 
     if track:
-        manifest.write(path=track)
+        # We should record files that we actually copied.
+        # It is too late to expand wildcards when the track file is read.
+        manifest.write(path=track, expand_pattern=True)
 
     return result
 

python/mozbuild/mozpack/manifests.py

@@ -228,7 +228,7 @@ def _decode_field_entry(self, data):
         """
         return json.loads(data)
 
-    def write(self, path=None, fileobj=None):
+    def write(self, path=None, fileobj=None, expand_pattern=False):
         """Serialize this manifest to a file or file object.
 
         If path is specified, that file will be written to. If fileobj is specified,
@@ -242,10 +242,21 @@ def write(self, path=None, fileobj=None):
             for dest in sorted(self._dests):
                 entry = self._dests[dest]
 
-                parts = ['%d' % entry[0], dest]
-                parts.extend(entry[1:])
-                fh.write('%s\n' % self.FIELD_SEPARATOR.join(
-                    p.encode('utf-8') for p in parts))
+                if expand_pattern and entry[0] in (self.PATTERN_SYMLINK, self.PATTERN_COPY):
+                    type, base, pattern, dest = entry
+                    type = self.SYMLINK if type == self.PATTERN_SYMLINK else self.COPY
+                    finder = FileFinder(base)
+                    paths = [f[0] for f in finder.find(pattern)]
+                    for path in paths:
+                        source = mozpath.join(base, path)
+                        parts = ['%d' % type, mozpath.join(dest, path), source]
+                        fh.write('%s\n' % self.FIELD_SEPARATOR.join(
+                            p.encode('utf-8') for p in parts))
+                else:
+                    parts = ['%d' % entry[0], dest]
+                    parts.extend(entry[1:])
+                    fh.write('%s\n' % self.FIELD_SEPARATOR.join(
+                        p.encode('utf-8') for p in parts))
 
     def add_symlink(self, source, dest):
         """Add a symlink to this manifest.
@@ -289,15 +300,15 @@ def add_pattern_symlink(self, base, pattern, dest):
 
            <base>/foo/bar.h -> <dest>/foo/bar.h
         """
-        self._add_entry(mozpath.join(base, pattern, dest),
+        self._add_entry(mozpath.join(dest, pattern),
             (self.PATTERN_SYMLINK, base, pattern, dest))
 
     def add_pattern_copy(self, base, pattern, dest):
         """Add a pattern match that results in copies.
 
         See ``add_pattern_symlink()`` for usage.
         """
-        self._add_entry(mozpath.join(base, pattern, dest),
+        self._add_entry(mozpath.join(dest, pattern),
             (self.PATTERN_COPY, base, pattern, dest))
 
     def add_preprocess(self, source, dest, deps, marker='#', defines={},

python/mozbuild/mozpack/test/test_manifests.py

@@ -156,6 +156,29 @@ def test_or(self):
         self.assertIn('s_dest2', m1)
         self.assertIn('c_dest2', m1)
 
+    def test_write_expand_pattern(self):
+        source = self.tmppath('source')
+        os.mkdir(source)
+        os.mkdir('%s/base' % source)
+        os.mkdir('%s/base/foo' % source)
+
+        with open('%s/base/foo/file1' % source, 'a'):
+            pass
+
+        with open('%s/base/foo/file2' % source, 'a'):
+            pass
+
+        m = InstallManifest()
+        m.add_pattern_link('%s/base' % source, '**', 'dest')
+
+        track = self.tmppath('track')
+        m.write(path=track, expand_pattern=True)
+
+        m = InstallManifest(path=track)
+        self.assertEqual([dest for dest in m._dests],
+                         ['dest/foo/file1', 'dest/foo/file2'])
+
+
     def test_copier_application(self):
         dest = self.tmppath('dest')
         os.mkdir(dest)