Initial Changes for Rebuilt CI (apple#1564)

* GitLab config: remove old dependencies * GitLab Config: Remove Linux * Fix issue found by flake8 + minor clean ups * xfail a unit test
RobertRiachi · Aug 3, 2022 · 49ba5ba · 49ba5ba
1 parent 4c9a191
commit 49ba5ba
Show file tree

Hide file tree

Showing 5 changed files with 33 additions and 142 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -12,65 +12,22 @@ stages:
 
 check_python_flake8:
   tags:
-    - docker
-  image: "python:3.7"
+    - macos12
   stage: check
   script:
     - python -m pip install --upgrade pip
     - pip install flake8
     - flake8 ./coremltools --count --select=E9,F5,F63,F7,F82 --show-source --statistics
 
-########################################################################
-#
-#                         linux - Build
-#
-########################################################################
-
-.build_linux: &build_linux
-  tags:
-    - docker
-  stage: build
-  script:
-    - export PATH=$PATH:/opt/anaconda/bin/
-    - zsh -e scripts/build.sh --num-procs=4 --python=$PYTHON --dist
-  artifacts:
-    expire_in: 2 weeks
-    paths:
-      - build/dist/
-
-build_wheel_linux_py36:
-  <<: *build_linux
-  image: registry.gitlab.com/zach_nation/coremltools/build-image-ubuntu-18.04:1.0.0
-  variables:
-    PYTHON: "3.6"
-
-build_wheel_linux_py37:
-  <<: *build_linux
-  image: registry.gitlab.com/zach_nation/coremltools/build-image-ubuntu-18.04:1.0.0
-  variables:
-    PYTHON: "3.7"
-
-build_wheel_linux_py38:
-  <<: *build_linux
-  image: registry.gitlab.com/zach_nation/coremltools/build-image-ubuntu-18.04:1.0.0
-  variables:
-    PYTHON: "3.8"
-
-build_wheel_linux_py39:
-  <<: *build_linux
-  image: registry.gitlab.com/zach_nation/coremltools/build-image-ubuntu-18.04:1.0.0
-  variables:
-    PYTHON: "3.9"
-
 #########################################################################
 ##
-##                         macOS - Build
+##                         Build
 ##
 #########################################################################
 
 .build_macos: &build_macos
     tags:
-       - macos10.15
+       - macos12
     stage: build
     script:
       - zsh -e scripts/build.sh --num-procs=4 --python=$PYTHON --dist
@@ -101,43 +58,7 @@ build_wheel_macos_py39:
 
 #########################################################################
 ##
-##                         linux - Test
-##
-#########################################################################
-.test_linux_pkg: &test_linux_pkg
-  stage: test
-  script:
-    - export PATH=$PATH:/opt/anaconda/bin/
-    - zsh -e scripts/test.sh --wheel-path=${WHEEL_PATH} --python=${PYTHON}
-      --test-package=${TEST_PACKAGE} --fast
-
-test_linux_py37_coremltools_test:
-  <<: *test_linux_pkg
-  tags:
-    - docker
-  image: registry.gitlab.com/zach_nation/coremltools/build-image-ubuntu-18.04:1.0.0
-  dependencies:
-    - build_wheel_linux_py37
-  variables:
-    WHEEL_PATH: build/dist/coremltools-*-none-manylinux1_x86_64.whl
-    TEST_PACKAGE: coremltools.test
-    PYTHON: "3.7"
-
-test_linux_py37_pytorch:
-  <<: *test_linux_pkg
-  tags:
-    - docker
-  image: registry.gitlab.com/zach_nation/coremltools/build-image-ubuntu-18.04:1.0.0
-  dependencies:
-    - build_wheel_linux_py37
-  variables:
-    PYTHON: "3.7"
-    TEST_PACKAGE: coremltools.converters.mil.frontend.torch
-    WHEEL_PATH: build/dist/coremltools-*-none-manylinux1_x86_64.whl
-
-#########################################################################
-##
-##                         macOS - Test
+##                         Test
 ##
 #########################################################################
 .test_macos_pkg: &test_macos_pkg
@@ -154,101 +75,72 @@ test_linux_py37_pytorch:
      - zsh -e scripts/test.sh --wheel-path=${WHEEL_PATH} --python=${PYTHON}
        --test-package=${TEST_PACKAGE} --requirements=${REQUIREMENTS} --fast
 
-test_macos11_py37_coremltools_test:
+test_py37_coremltools_test:
   <<: *test_macos_pkg
   tags:
-    - macos11
+    - macos12
   dependencies:
     - build_wheel_macos_py37
   variables:
     WHEEL_PATH: build/dist/*cp37*10_15*
     TEST_PACKAGE: coremltools.test
     PYTHON: "3.7"
 
-test_macos11_py37_pytorch:
+test_py37_pytorch:
   <<: *test_macos_pkg
   tags:
-    - macos11
+    - macos12
   dependencies:
     - build_wheel_macos_py37
   variables:
     PYTHON: "3.7"
     TEST_PACKAGE: coremltools.converters.mil.frontend.torch
     WHEEL_PATH: build/dist/*cp37*10_15*
 
-test_macos11_py37_tf1:
+test_py37_tf1:
   <<: *test_macos_pkg
   tags:
-    - macos11
+    - macos12
   dependencies:
     - build_wheel_macos_py37
   variables:
     PYTHON: "3.7"
     TEST_PACKAGE: coremltools.converters.mil.frontend.tensorflow
     WHEEL_PATH: build/dist/*cp37*10_15*
 
-test_macos11_py37_mil:
+test_py37_mil:
   <<: *test_macos_pkg
   tags:
-    - macos11
+    - macos12
   dependencies:
     - build_wheel_macos_py37
   variables:
     PYTHON: "3.7"
     TEST_PACKAGE: coremltools.converters.mil.mil
     WHEEL_PATH: build/dist/*cp37*10_15*
 
-#########################################################################
-##
-##                macOS - Smoke Test on older versions
-##
-#########################################################################
-
-test_macos11_py38_coremltools_smoke_test:
-  <<: *test_macos_pkg
-  tags:
-    - macos11
-  dependencies:
-    - build_wheel_macos_py38
-  variables:
-    WHEEL_PATH: build/dist/*cp38*10_15*
-    TEST_PACKAGE: coremltools.test.neural_network.test_simple_nn_inference
-    PYTHON: "3.8"
-
-test_macos10_15_py38_coremltools_smoke_test:
-  <<: *test_macos_pkg
-  tags:
-    - macos10.15
-  dependencies:
-    - build_wheel_macos_py38
-  variables:
-    WHEEL_PATH: build/dist/*cp38*10_15*
-    TEST_PACKAGE: coremltools.test.neural_network.test_simple_nn_inference
-    PYTHON: "3.8"
-
 #########################################################################
 ##
 ##                        Make docs
 ##
 #########################################################################
 build_documentation:
   tags:
-    - docker
+    - macos12
   stage: test
-  image: registry.gitlab.com/zach_nation/coremltools/build-image-ubuntu-18.04:1.0.0
   script:
     - export PATH=$PATH:/opt/anaconda/bin/
     - bash -e scripts/build_docs.sh --wheel-path=${WHEEL_PATH} --python=${PYTHON}
   dependencies:
-    - build_wheel_linux_py37
+    - build_wheel_macos_py39
   artifacts:
     when: always
     expire_in: 2 weeks
     paths:
       - _build/html/
   variables:
-    WHEEL_PATH: build/dist/coremltools*cp37-none-manylinux1_x86_64.whl
-    PYTHON: "3.7"
+    WHEEL_PATH: build/dist/coremltools*cp39-none-macosx_10_15_x86_64.whl
+    PYTHON: "3.9"
 
 #########################################################################
 ##
@@ -257,22 +149,16 @@ build_documentation:
 #########################################################################
 collect_artifacts:
   tags:
-    - docker
-  services:
-    - docker:18.09-dind
-  image: alpine:latest
+    - macos12
   stage: collect
   script:
     echo "Collect artifacts (wheels and documentation)"
   dependencies:
-    - build_wheel_linux_py35
-    - build_wheel_linux_py36
-    - build_wheel_linux_py37
-    - build_wheel_linux_py38
-    - build_wheel_macos_py35
     - build_wheel_macos_py36
     - build_wheel_macos_py37
     - build_wheel_macos_py38
+    - build_wheel_macos_py39
+    - build_documentation
   artifacts:
     expire_in: 2 weeks
     paths:

diff --git a/coremltools/converters/mil/frontend/milproto/load.py b/coremltools/converters/mil/frontend/milproto/load.py
@@ -3,8 +3,8 @@
 # Use of this source code is governed by a BSD-3-clause license that can be
 # found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
 
-import os
 import logging
+import os
 
 import numpy as np
 
@@ -294,9 +294,10 @@ def _load_operation(context, op_spec):
         for param_name, argument in op_spec.inputs.items():
             vars = []
             for binding in argument.arguments:
-                if binding.WhichOneof("binding") == "name":
+                binding_type = binding.WhichOneof("binding")
+                if binding_type == "name":
                     vars.append(context.get_var_from_name(binding.name))
-                elif binding.WhichOneof("binding") == "value":
+                elif binding_type == "value":
                     # We only support the list value for now (for the classifier use case)
                     value_spec = binding.value
                     assert value_spec.WhichOneof("value") == "immediateValue"

diff --git a/coremltools/converters/mil/frontend/torch/test/test_torch_ops.py b/coremltools/converters/mil/frontend/torch/test/test_torch_ops.py
@@ -194,6 +194,9 @@ class TestMean(TorchBaseTest):
         backends,
     )
     def test_with_flexible_shape(self, backend):
+        if backend[0] == 'mlprogram' and _macos_version() < (13, 0):
+            pytest.xfail("Issue fixed in iOS16/macOS13: https://github.com/apple/coremltools/issues/1420")
+
         class Model(nn.Module):
             def __init__(self):
                 super().__init__()

diff --git a/coremltools/converters/mil/mil/ops/defs/iOS15/tensor_transformation.py b/coremltools/converters/mil/mil/ops/defs/iOS15/tensor_transformation.py
@@ -4,6 +4,7 @@
 #  found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
 
 import logging
+
 import numpy as np
 import sympy as sm
 
@@ -34,10 +35,10 @@
     SYMBOL,
     VALUE
 )
-
 from coremltools.converters.mil.mil.ops.defs._op_reqs import register_op
 from coremltools.converters.mil.mil.ops.defs._utils import solve_slice_by_index_shape
 
+
 @register_op
 class depth_to_space(Operation):
     """
@@ -838,7 +839,7 @@ def type_inference(self):
                              "len(block_shape){} + 2! Got {}".format(self.block_shape.val, self.x.rank))
 
         if not is_symbolic(b) and  b % np.prod(block_shape) != 0:
-            msg = ("Batch size must be perfectly divided by the product of block_shape. Got batch size {}, and block_shape"
+            msg = ("Batch size must be perfectly divided by the product of block_shape. Got batch size {}, and block_shape {}."
             ).format(b, block_shape)
             raise ValueError(msg)
 

diff --git a/coremltools/converters/mil/mil/passes/compression_passes.py b/coremltools/converters/mil/mil/passes/compression_passes.py
@@ -1,7 +1,7 @@
-# Copyright (c) 2022, Apple Inc. All rights reserved.
+#  Copyright (c) 2022, Apple Inc. All rights reserved.
 #
-# Use of this source code is governed by a BSD-3-clause license that can be
-# found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
+#  Use of this source code is governed by a BSD-3-clause license that can be
+#  found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
 
 from enum import Enum
 import logging as _logging
@@ -164,7 +164,7 @@ def __init__(self, nbits, fake_compression=False, op_selector=None, mode="kmeans
             raise ValueError(msg)
 
         if self.nbits is not None and self.nbits not in (1, 2, 4, 6, 8):
-            raise ValueError("Invalid value of nbits ({}) for palettization. Supported bits are {1, 2, 4, 6, 8}".format(nbits))
+            raise ValueError("Invalid value of nbits ({}) for palettization. Supported bits are {{1, 2, 4, 6, 8}}".format(nbits))
 
         if (self.mode == "CUSTOM") ^ (lut_function is not None):
             msg = "lut_function must be None if mode is not custom, and that it cannot be None when the mode is custom."