Merge pull request #91 from google/upstream-1533848866

Support foreign layers

client/v1/docker_image_.py

@@ -108,6 +108,10 @@ def uncompressed_layer(self, layer_id):
     unzipped = f.read()
     return unzipped
 
+  def diff_id(self, digest):
+    """diff_id only exist in schema v22."""
+    return None
+
   # pytype: disable=bad-return-type
   @abc.abstractmethod
   def ancestry(self, layer_id):

client/v2/docker_image_.py

@@ -90,6 +90,10 @@ def uncompressed_blob(self, digest):
     f = gzip.GzipFile(mode='rb', fileobj=buf)
     return f.read()
 
+  def diff_id(self, digest):
+    """diff_id only exist in schema v22."""
+    return None
+
   # __enter__ and __exit__ allow use as a context manager.
   @abc.abstractmethod
   def __enter__(self):

client/v2/v1_compat_.py

@@ -107,6 +107,10 @@ def layer(self, layer_id):
     v2_digest = self._v1_to_v2.get(layer_id)
     return self._v2_image.blob(v2_digest)
 
+  def diff_id(self, digest):
+    """Override."""
+    return self._v2_image.diff_id(self._v1_to_v2.get(digest))
+
   def ancestry(self, layer_id):
     """Override."""
     index = self._v1_ancestry.index(layer_id)

client/v2_2/docker_http_.py

@@ -42,11 +42,15 @@
 MANIFEST_SCHEMA2_MIME = 'application/vnd.docker.distribution.manifest.v2+json'
 MANIFEST_LIST_MIME = 'application/vnd.docker.distribution.manifest.list.v2+json'
 LAYER_MIME = 'application/vnd.docker.image.rootfs.diff.tar.gzip'
+FOREIGN_LAYER_MIME = 'application/vnd.docker.image.rootfs.foreign.diff.tar.gzip'
 CONFIG_JSON_MIME = 'application/vnd.docker.container.image.v1+json'
 
 OCI_MANIFEST_MIME = 'application/vnd.oci.image.manifest.v1+json'
 OCI_IMAGE_INDEX_MIME = 'application/vnd.oci.image.index.v1+json'
-OCI_LAYER_MIME = 'application/vnd.oci.image.layer.v1.tar+gzip'
+OCI_LAYER_MIME = 'application/vnd.oci.image.layer.v1.tar'
+OCI_GZIP_LAYER_MIME = 'application/vnd.oci.image.layer.v1.tar+gzip'
+OCI_NONDISTRIBUTABLE_LAYER_MIME = 'application/vnd.oci.image.layer.nondistributable.v1.tar'  # pylint disable=line-too-long
+OCI_NONDISTRIBUTABLE_GZIP_LAYER_MIME = 'application/vnd.oci.image.layer.nondistributable.v1.tar+gzip'  # pylint disable=line-too-long
 OCI_CONFIG_JSON_MIME = 'application/vnd.oci.image.config.v1+json'
 
 MANIFEST_SCHEMA1_MIMES = [MANIFEST_SCHEMA1_MIME, MANIFEST_SCHEMA1_SIGNED_MIME]
@@ -59,6 +63,12 @@
 # OCI Image Index and Manifest List are compatible formats.
 MANIFEST_LIST_MIMES = [OCI_IMAGE_INDEX_MIME, MANIFEST_LIST_MIME]
 
+# Docker & OCI layer mime types indicating foreign/non-distributable layers.
+NON_DISTRIBUTABLE_LAYER_MIMES = [
+    FOREIGN_LAYER_MIME, OCI_NONDISTRIBUTABLE_LAYER_MIME,
+    OCI_NONDISTRIBUTABLE_GZIP_LAYER_MIME
+]
+
 
 class Diagnostic(object):
   """Diagnostic encapsulates a Registry v2 diagnostic message.

client/v2_2/docker_image_.py

@@ -61,6 +61,17 @@ def blob_set(self):
     """The unique set of blobs that compose to create the filesystem."""
     return set(self.fs_layers() + [self.config_blob()])
 
+  def distributable_blob_set(self):
+    """The unique set of blobs which are distributable."""
+    manifest = json.loads(self.manifest())
+    distributable_blobs = {
+        x['digest']
+        for x in reversed(manifest['layers'])
+        if x['mediaType'] not in docker_http.NON_DISTRIBUTABLE_LAYER_MIMES
+    }
+    distributable_blobs.add(self.config_blob())
+    return distributable_blobs
+
   def digest(self):
     """The digest of the manifest."""
     return docker_digest.SHA256(self.manifest().encode('utf8'))
@@ -119,6 +130,13 @@ def _diff_id_to_digest(self, diff_id):
         return this_digest
     raise ValueError('Unmatched "diff_id": "%s"' % diff_id)
 
+  def digest_to_diff_id(self, digest):
+    for (this_digest, this_diff_id) in six.moves.zip(self.fs_layers(),
+                                                     self.diff_ids()):
+      if this_digest == digest:
+        return this_diff_id
+    raise ValueError('Unmatched "digest": "%s"' % digest)
+
   def layer(self, diff_id):
     """Like `blob()`, but accepts the `diff_id` instead.
 
@@ -424,7 +442,7 @@ def _content(self,
     # https://mail.python.org/pipermail/python-bugs-list/2015-March/265999.html
     # so instead of locking, just open the tarfile for each file
     # we want to read.
-    with tarfile.open(name=self._tarball, mode='r:') as tar:
+    with tarfile.open(name=self._tarball, mode='r') as tar:
       try:
         # If the layer is compressed and we need to return compressed
         # or if it's uncompressed and we need to return uncompressed
@@ -477,16 +495,41 @@ def _populate_manifest_and_blobs(self):
     }
 
     blob_names = {}
-    for layer in self._layers:
-      content = self._gzipped_content(layer)
-      name = docker_digest.SHA256(content)
+
+    config = json.loads(self.config_file())
+    diff_ids = config['rootfs']['diff_ids']
+
+    for i, layer in enumerate(self._layers):
+      name = None
+      diff_id = diff_ids[i]
+      media_type = docker_http.LAYER_MIME
+      size = 0
+      urls = []
+
+      if diff_id in self._layer_sources:
+        # _layer_sources contains foreign layers from the base image
+        name = self._layer_sources[diff_id]['digest']
+        media_type = self._layer_sources[diff_id]['mediaType']
+        size = self._layer_sources[diff_id]['size']
+        if 'urls' in self._layer_sources[diff_id]:
+          urls = self._layer_sources[diff_id]['urls']
+      else:
+        content = self._gzipped_content(layer)
+        name = docker_digest.SHA256(content)
+        size = len(content)
+
       blob_names[name] = layer
-      manifest['layers'].append({
+
+      layer_manifest = {
           'digest': name,
-          # TODO(user): Do we need to sniff the file to detect this?
-          'mediaType': docker_http.LAYER_MIME,
-          'size': len(content),
-      })
+          'mediaType': media_type,
+          'size': size,
+      }
+
+      if urls:
+        layer_manifest['urls'] = urls
+
+      manifest['layers'].append(layer_manifest)
 
     with self._lock:
       self._manifest = manifest
@@ -555,6 +598,7 @@ def __enter__(self):
 
     config = None
     layers = []
+    layer_sources = []
     # Find the right entry, either:
     # 1) We were supplied with an image name, which we must find in an entry's
     #   RepoTags, or
@@ -572,13 +616,15 @@ def __enter__(self):
       if not self._name or str(self._name) in (entry.get('RepoTags') or []):
         config = entry.get('Config')
         layers = entry.get('Layers', [])
+        layer_sources = entry.get('LayerSources', {})
 
     if not config:
       raise ValueError('Unable to find %s in provided tarball.' % self._name)
 
     # Metadata from the tarball's configuration we need to construct the image.
     self._config_file = config
     self._layers = layers
+    self._layer_sources = layer_sources
 
     # We populate "manifest" and "blobs" lazily for two reasons:
     # 1) Allow use of this library for reading the config_file() from the image
@@ -614,15 +660,19 @@ class FromDisk(DockerImage):
         path to a file containing the second element's sha256.
         The second element is the .tar of a filesystem layer.
     legacy_base: Optionally, the path to a legacy base image in FromTarball form
+    foreign_layers_manifest: Optionally a tar manifest from the base
+        image that describes the ForeignLayers needed by this image.
   """
 
   def __init__(self,
                config_file,
                layers,
                uncompressed_layers = None,
-               legacy_base = None):
+               legacy_base = None,
+               foreign_layers_manifest = None):
     self._config = config_file
     self._manifest = None
+    self._foreign_layers_manifest = foreign_layers_manifest
     self._layers = []
     self._layer_to_filename = {}
     for (name_file, content_file) in layers:
@@ -649,6 +699,19 @@ def _populate_manifest(self):
     base_layers = []
     if self._legacy_base:
       base_layers = json.loads(self._legacy_base.manifest())['layers']
+    elif self._foreign_layers_manifest:
+      # Manifest files found in tar files are actually a json list.
+      # This code iterates through that collection and appends any foreign
+      # layers described in the order found in the config file.
+      foreign_layers_list = json.loads(self._foreign_layers_manifest)
+      for foreign_layers in foreign_layers_list:
+        if 'LayerSources' in foreign_layers:
+          config = json.loads(self._config)
+          layer_sources = foreign_layers['LayerSources']
+          for diff_id in config['rootfs']['diff_ids']:
+            if diff_id in layer_sources:
+              base_layers.append(layer_sources[diff_id])
+
     # TODO(user): Update mimes here for oci_compat.
     self._manifest = json.dumps(
         {

client/v2_2/docker_session_.py

@@ -305,14 +305,14 @@ def upload(self,
         with child:
           self.upload(child, use_digest=True)
     elif self._threads == 1:
-      for digest in image.blob_set():
+      for digest in image.distributable_blob_set():
         self._upload_one(image, digest)
     else:
       with concurrent.futures.ThreadPoolExecutor(
           max_workers=self._threads) as executor:
         future_to_params = {
             executor.submit(self._upload_one, image, digest): (image, digest)
-            for digest in image.blob_set()
+            for digest in image.distributable_blob_set()
         }
         for future in concurrent.futures.as_completed(future_to_params):
           future.result()

client/v2_2/save_.py

@@ -37,8 +37,12 @@
 
 
 def _diff_id(v1_img, blob):
-  unzipped = v1_img.uncompressed_layer(blob)
-  return docker_digest.SHA256(unzipped)
+  try:
+    unzipped = v1_img.uncompressed_layer(blob)
+    return docker_digest.SHA256(unzipped)
+  except IOError:
+    # For foreign layers, we do not have the layer.tar
+    return v1_img.diff_id(blob)
 
 
 def multi_image_tarball(
@@ -72,6 +76,8 @@ def add_file(filename, contents):
   #           ancestry ordering.
   #  - RepoTags: the list of tags to apply to this image once it
   #             is loaded.
+  #  - LayerSources: optionally declare foreign layers declared in
+  #                  the base image
   manifests = []
 
   for (tag, image) in six.iteritems(tag_to_image):
@@ -89,7 +95,7 @@ def add_file(filename, contents):
       tag_to_v1_image[tag] = v1_img
 
     # Add the manifests entry for this image.
-    manifests.append({
+    manifest = {
         'Config':
             digest + '.json',
         'Layers': [
@@ -100,7 +106,14 @@ def add_file(filename, contents):
             if _diff_id(v1_img, layer_id) in diffs
         ],
         'RepoTags': [str(tag)]
-    })
+    }
+
+    input_manifest = json.loads(image.manifest())
+
+    if 'LayerSources' in input_manifest:
+      manifest['LayerSources'] = input_manifest['LayerSources']
+
+    manifests.append(manifest)
 
   # v2.2 tarballs are a superset of v1 tarballs, so delegate
   # to v1 to save itself.

client/v2_2/v2_compat_.py

@@ -198,7 +198,7 @@ def _ProcessImage(self):
     fs_layers = []
     v1_histories = []
     for v1_layer_index, history in enumerate(histories):
-      digest, v2_layer_index = self._GetSchema1LayerDigest(
+      digest, media_type, v2_layer_index = self._GetSchema1LayerDigest(
           history, layers, v1_layer_index, v2_layer_index)
 
       if v1_layer_index != layer_count - 1:
@@ -209,7 +209,7 @@ def _ProcessImage(self):
         v1_compatibility = self._BuildV1CompatibilityForTopLayer(
             layer_id, parent, history, config)
       parent = layer_id
-      fs_layers = [{'blobSum': digest}] + fs_layers
+      fs_layers = [{'blobSum': digest, 'mediaType': media_type}] + fs_layers
       v1_histories = [{'v1Compatibility': v1_compatibility}] + v1_histories
 
     manifest_schema1 = {
@@ -281,9 +281,13 @@ def _GetSchema1LayerDigest(
       self, history, layers,
       v1_layer_index, v2_layer_index):
     if 'empty_layer' in history:
-      return (EMPTY_TAR_DIGEST, v2_layer_index)
+      return (EMPTY_TAR_DIGEST, docker_http.LAYER_MIME, v2_layer_index)
     else:
-      return (layers[v2_layer_index]['digest'], v2_layer_index + 1)
+      return (
+          layers[v2_layer_index]['digest'],
+          layers[v2_layer_index]['mediaType'],
+          v2_layer_index + 1
+      )
 
   def manifest(self):
     """Override."""
@@ -296,6 +300,10 @@ def uncompressed_blob(self, digest):
       return super(V2FromV22, self).uncompressed_blob(EMPTY_TAR_DIGEST)
     return self._v2_2_image.uncompressed_blob(digest)
 
+  def diff_id(self, digest):
+    """Gets v22 diff_id."""
+    return self._v2_2_image.digest_to_diff_id(digest)
+
   def blob(self, digest):
     """Override."""
     if digest == EMPTY_TAR_DIGEST: