Merge pull request #131 from google/upstream-1543610079

Replace --platform in pullers with specific args, use custem docker client config dir

README.md

@@ -33,8 +33,13 @@ $ bazel run @containerregistry//:puller.par -- --help
 ```
 
 ```
-usage: puller.par [-h] --name NAME --directory DIRECTORY [--platform PLATFORM]
-                  [--stderrthreshold STDERRTHRESHOLD]
+usage: puller.par [-h] --name NAME --directory DIRECTORY [--os OS]
+                   [--os-version OS_VERSION]
+                   [--os-features [OS_FEATURES [OS_FEATURES ...]]]
+                   [--architecture ARCHITECTURE] [--variant VARIANT]
+                   [--features [FEATURES [FEATURES ...]]]
+                   [--client-config-dir CLIENT_CONFIG_DIR]
+                   [--stderrthreshold STDERRTHRESHOLD]
 
 Pull images from a Docker Registry, faaaaast.
 
@@ -44,10 +49,29 @@ optional arguments:
                         Supports fully-qualified tag or digest references.
   --directory DIRECTORY
                         Where to save the image's files.
-  --platform PLATFORM   Which platform image to pull for multi-platform
-                        manifest lists. Formatted as os/arch.
+  --os OS               For multi-platform manifest lists, specifies the
+                        operating system.
+  --os-version OS_VERSION
+                        For multi-platform manifest lists, specifies the
+                        operating system version.
+  --os-features [OS_FEATURES [OS_FEATURES ...]]
+                        For multi-platform manifest lists, specifies operating
+                        system features.
+  --architecture ARCHITECTURE
+                        For multi-platform manifest lists, specifies the CPU
+                        architecture.
+  --variant VARIANT     For multi-platform manifest lists, specifies the CPU
+                        variant.
+  --features [FEATURES [FEATURES ...]]
+                        For multi-platform manifest lists, specifies CPU
+                        features.
+  --client-config-dir CLIENT_CONFIG_DIR
+                        The path to the directory where the client
+                        configuration files are located. Overiddes the value
+                        from DOCKER_CONFIG
   --stderrthreshold STDERRTHRESHOLD
                         Write log events at or above this level to stderr.
+
 ```
 
 ## pusher.par
@@ -58,9 +82,10 @@ $ bazel run @containerregistry//:pusher.par -- --help
 
 ```
 usage: pusher.par [-h] --name NAME [--tarball TARBALL] [--config CONFIG]
-                  [--manifest MANIFEST] [--digest DIGEST] [--layer LAYER]
-                  [--stamp-info-file STAMP_INFO_FILE] [--oci]
-                  [--stderrthreshold STDERRTHRESHOLD]
+                   [--manifest MANIFEST] [--digest DIGEST] [--layer LAYER]
+                   [--stamp-info-file STAMP_INFO_FILE] [--oci]
+                   [--client-config-dir CLIENT_CONFIG_DIR]
+                   [--stderrthreshold STDERRTHRESHOLD]
 
 Push images to a Docker Registry, faaaaaast.
 
@@ -76,8 +101,13 @@ optional arguments:
                         A list of files from which to read substitutions to
                         make in the provided --name, e.g. {BUILD_USER}
   --oci                 Push the image with an OCI Manifest.
+  --client-config-dir CLIENT_CONFIG_DIR
+                        The path to the directory where the client
+                        configuration files are located. Overiddes the value
+                        from DOCKER_CONFIG
   --stderrthreshold STDERRTHRESHOLD
                         Write log events at or above this level to stderr.
+
 ```
 
 ## importer.par

client/docker_creds_.py

@@ -231,11 +231,30 @@ def _GetConfigDirectory():
 class _DefaultKeychain(Keychain):
   """This implements the default docker credential resolution."""
 
+  def __init__(self):
+    # Store a custom directory to get the Docker configuration JSON from
+    self._config_dir = None
+    # Name of the docker configuration JSON file to look for in the
+    # configuration directory
+    self._config_file = 'config.json'
+
+  def setCustomConfigDir(self, config_dir):
+    # Override the configuration directory where the docker configuration
+    # JSON is searched for
+    if not os.path.isdir(config_dir):
+      raise Exception('Attempting to override docker configuration directory'
+                      ' to invalid directory: {}'.format(config_dir))
+    self._config_dir = config_dir
+
   def Resolve(self, name):
     # TODO(user): Consider supporting .dockercfg, which was used prior
     # to Docker 1.7 and consisted of just the contents of 'auths' below.
     logging.info('Loading Docker credentials for repository %r', str(name))
-    config_file = os.path.join(_GetConfigDirectory(), 'config.json')
+    config_file = None
+    if self._config_dir is not None:
+      config_file = os.path.join(self._config_dir, self._config_file)
+    else:
+      config_file = os.path.join(_GetConfigDirectory(), self._config_file)
     try:
       with io.open(config_file, u'r', encoding='utf8') as reader:
         cfg = json.loads(reader.read())

def.bzl

@@ -12,8 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
 def repositories():
-    native.new_http_archive(
+    """Load bazel dependencies."""
+
+    http_archive(
         name = "httplib2",
         url = "https://codeload.github.com/httplib2/httplib2/tar.gz/v0.11.3",
         sha256 = "d9f568c183d1230f271e9c60bd99f3f2b67637c3478c9068fea29f7cca3d911f",
@@ -29,7 +34,7 @@ py_library(
     )
 
     # Used by oauth2client
-    native.new_http_archive(
+    http_archive(
         name = "six",
         url = "https://pypi.python.org/packages/source/s/six/six-1.9.0.tar.gz",
         sha256 = "e24052411fc4fbd1f672635537c3fc2330d9481b18c0317695b46259512c91d5",
@@ -51,7 +56,7 @@ py_library(
     )
 
     # Used for authentication in containerregistry
-    native.new_http_archive(
+    http_archive(
         name = "oauth2client",
         url = "https://codeload.github.com/google/oauth2client/tar.gz/v4.0.0",
         sha256 = "7230f52f7f1d4566a3f9c3aeb5ffe2ed80302843ce5605853bee1f08098ede46",
@@ -70,7 +75,7 @@ py_library(
     )
 
     # Used for parallel execution in containerregistry
-    native.new_http_archive(
+    http_archive(
         name = "concurrent",
         url = "https://codeload.github.com/agronholm/pythonfutures/tar.gz/3.0.5",
         sha256 = "a7086ddf3c36203da7816f7e903ce43d042831f41a9705bc6b4206c574fcb765",
@@ -85,7 +90,7 @@ py_library(
     )
 
     # For packaging python tools.
-    native.git_repository(
+    git_repository(
         name = "subpar",
         remote = "https://github.com/google/subpar",
         commit = "07ff5feb7c7b113eea593eb6ec50b51099cf0261",

puller_test.sh

@@ -29,14 +29,15 @@ function test_puller() {
 
 test_puller_multiplatform() {
   local image=$1
-  local platform=$2
-  local expected_digest=$3
+  local expected_digest=$2
+  shift 2
+  local platform_args=$@
 
   local tmpdir=$(mktemp -d)
 
-  echo "TESTING: ${image} platform ${platform}"
+  echo "TESTING: ${image} ${platform_args}"
 
-  puller.par --name="${image}" --directory="${tmpdir}" --platform="${platform}"
+  puller.par --name="${image}" --directory="${tmpdir}" ${platform_args}
 
   digest=$(cat "${tmpdir}/digest")
   rm -rf "${tmpdir}"
@@ -109,10 +110,18 @@ test_image gcr.io/google-containers/pause@sha256:9ce5316f9752b8347484ab0f6778573
 test_image index.docker.io/library/busybox@sha256:1669a6aa7350e1cdd28f972ddad5aceba2912f589f19a090ac75b7083da748db
 
 # Test pulling manifest list explicitly specifying a platform
-test_puller_multiplatform gcr.io/google-containers/pause:3.1 linux/amd64 \
-  sha256:59eec8837a4d942cc19a52b8c09ea75121acc38114a2c68b98983ce9356b8610
+test_puller_multiplatform gcr.io/google-containers/pause:3.1 \
+  sha256:59eec8837a4d942cc19a52b8c09ea75121acc38114a2c68b98983ce9356b8610 \
+  --os linux --architecture amd64
 
-test_puller_multiplatform gcr.io/google-containers/pause:3.1 linux/ppc64le \
-  sha256:bcf9771c0b505e68c65440474179592ffdfa98790eb54ffbf129969c5e429990
+test_puller_multiplatform gcr.io/google-containers/pause:3.1 \
+  sha256:bcf9771c0b505e68c65440474179592ffdfa98790eb54ffbf129969c5e429990 \
+  --os linux --architecture ppc64le
+
+test_puller_multiplatform index.docker.io/library/busybox:1.29.3 \
+  sha256:11a6b4baf996d8e52a332fbe7117aca1aae2b3068c7106f5b1065c16e8660895 \
+  --os linux --architecture arm --variant v5
+
+# TODO: add multiplatform test cases on --os-features and --features
 
 # TODO(user): Add an authenticated pull test.

tools/__init__.py

@@ -20,6 +20,10 @@
 setattr(x, 'patched', patched_)
 
 
+from containerregistry.tools import platform_args_
+setattr(x, 'platform_args', platform_args_)
+
+
 from containerregistry.tools import logging_setup_
 setattr(x, 'logging_setup', logging_setup_)
 

tools/docker_puller_.py

@@ -30,6 +30,7 @@
 from containerregistry.client.v2_2 import v2_compat
 from containerregistry.tools import logging_setup
 from containerregistry.tools import patched
+from containerregistry.tools import platform_args
 from containerregistry.transport import retry
 from containerregistry.transport import transport_pool
 
@@ -50,10 +51,7 @@
     '--tarball', action='store', help='Where to save the image tarball.',
     required=True)
 
-parser.add_argument(
-    '--platform', action='store', default='linux/amd64',
-    help=('Which platform image to pull for multi-platform manifest lists. '
-          'Formatted as os/arch.'))
+platform_args.AddArguments(parser)
 
 _DEFAULT_TAG = 'i-was-a-digest'
 
@@ -82,12 +80,6 @@ def main():
   args = parser.parse_args()
   logging_setup.Init(args=args)
 
-  if '/' not in args.platform:
-    logging.fatal('--platform must be specified in os/arch format.')
-    sys.exit(1)
-
-  os, arch = args.platform.split('/', 1)
-
   retry_factory = retry.Factory()
   retry_factory = retry_factory.WithSourceTransportCallable(httplib2.Http)
   transport = transport_pool.Http(retry_factory.Build, size=8)
@@ -120,10 +112,7 @@ def main():
       logging.info('Pulling manifest list from %r ...', name)
       with image_list.FromRegistry(name, creds, transport) as img_list:
         if img_list.exists():
-          platform = image_list.Platform({
-              'architecture': arch,
-              'os': os,
-          })
+          platform = platform_args.FromArgs(args)
           # pytype: disable=wrong-arg-types
           with img_list.resolve(platform) as default_child:
             save.tarball(_make_tag_if_digest(name), default_child, tar)

tools/fast_puller_.py

@@ -32,6 +32,7 @@
 from containerregistry.client.v2_2 import v2_compat
 from containerregistry.tools import logging_setup
 from containerregistry.tools import patched
+from containerregistry.tools import platform_args
 from containerregistry.transport import retry
 from containerregistry.transport import transport_pool
 
@@ -52,10 +53,13 @@
     '--directory', action='store', help='Where to save the image\'s files.',
     required=True)
 
+platform_args.AddArguments(parser)
+
 parser.add_argument(
-    '--platform', action='store', default='linux/amd64',
-    help=('Which platform image to pull for multi-platform manifest lists. '
-          'Formatted as os/arch.'))
+    '--client-config-dir',
+    action='store',
+    help='The path to the directory where the client configuration files are '
+    'located. Overiddes the value from DOCKER_CONFIG')
 
 _THREADS = 8
 
@@ -65,12 +69,6 @@ def main():
   args = parser.parse_args()
   logging_setup.Init(args=args)
 
-  if '/' not in args.platform:
-    logging.fatal('--platform must be specified in os/arch format.')
-    sys.exit(1)
-
-  os, arch = args.platform.split('/', 1)
-
   retry_factory = retry.Factory()
   retry_factory = retry_factory.WithSourceTransportCallable(httplib2.Http)
   transport = transport_pool.Http(retry_factory.Build, size=_THREADS)
@@ -80,6 +78,11 @@ def main():
   else:
     name = docker_name.Tag(args.name)
 
+  # If the user provided a client config directory, instruct the keychain
+  # resolver to use it to look for the docker client config
+  if args.client_config_dir is not None:
+    docker_creds.DefaultKeychain.setCustomConfigDir(args.client_config_dir)
+
   # OCI Image Manifest is compatible with Docker Image Manifest Version 2,
   # Schema 2. We indicate support for both formats by passing both media types
   # as 'Accept' headers.
@@ -102,10 +105,7 @@ def main():
     logging.info('Pulling manifest list from %r ...', name)
     with image_list.FromRegistry(name, creds, transport) as img_list:
       if img_list.exists():
-        platform = image_list.Platform({
-            'architecture': arch,
-            'os': os,
-        })
+        platform = platform_args.FromArgs(args)
         # pytype: disable=wrong-arg-types
         with img_list.resolve(platform) as default_child:
           save.fast(default_child, args.directory, threads=_THREADS)

tools/fast_pusher_.py

@@ -80,6 +80,12 @@
 parser.add_argument(
     '--oci', action='store_true', help='Push the image with an OCI Manifest.')
 
+parser.add_argument(
+    '--client-config-dir',
+    action='store',
+    help='The path to the directory where the client configuration files are '
+    'located. Overiddes the value from DOCKER_CONFIG')
+
 _THREADS = 8
 
 
@@ -145,6 +151,11 @@ def main():
     logging.fatal('--digest and --layer must have matching lengths.')
     sys.exit(1)
 
+  # If the user provided a client config directory, instruct the keychain
+  # resolver to use it to look for the docker client config
+  if args.client_config_dir is not None:
+    docker_creds.DefaultKeychain.setCustomConfigDir(args.client_config_dir)
+
   retry_factory = retry.Factory()
   retry_factory = retry_factory.WithSourceTransportCallable(httplib2.Http)
   transport = transport_pool.Http(retry_factory.Build, size=_THREADS)