Improves MANIFEST.in (#93)

xadupre · web-flow · commit d6d826cd2cb7 · 2023-10-06T11:17:12.000+02:00
* Improves MANIFEST.in

* setup

* disable check release

* update setup

* fix setup

* fix path

* fix command line to test a wheel

* update cmake messages

* fix wheels

* fix mypy

* update cmake

* fix cmake

* fix check function

* doc

* doc
diff --git a/.github/workflows/check-release.yml b/.github/workflows/check-release.yml
@@ -4,7 +4,7 @@ on:
   schedule:
     # every first day of the month
     - cron:  '0 0 1 * *'
-  push:
+  # push:
 
 jobs:
 
diff --git a/CHANGELOGS.rst b/CHANGELOGS.rst
@@ -4,6 +4,7 @@ Change Logs
 0.2.3
 +++++
 
+* :pr:`93`: validates the wheels in CI
 * :pr:`89`: add a function to merge models and update them if both have different opsets
 
 0.2.2
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,8 +1,12 @@
-recursive-include onnx_extended *.c *.cpp *.h *.pyx *.pxd *.pxi *.py _cmake *.cmake *.in *.txt *.sh
+recursive-include onnx_extended *.c *.cpp *.h *.pyx *.pxd *.pxi *.py
+recursive-include  _cmake *.cmake *.in *.txt *.sh *.in
 include pyproject.toml
 include MANIFEST.in
 include setup.cfg
 prune _doc
 prune _unittests
 exclude *.yml
 exclude *.git*
+# cython files to exclude
+exclude onnx_extended/ortcy/ortinf.cpp
+exclude onnx_extended/validation/cython/fp8.cpp
diff --git a/_cmake/CMakeLists.txt b/_cmake/CMakeLists.txt
@@ -86,9 +86,11 @@ include("finalize.cmake")
 #
 
 message(STATUS "---------------------------------------------------")
-message(STATUS "WRITE _CONFIG: '${ROOT_PROJECT_PATH}/onnx_extended/_config.py'")
 message(STATUS "CONTENT\n ${config_content}")
+message(STATUS "WRITE _CONFIG: '${ROOT_PROJECT_PATH}/onnx_extended/_config.py'")
 file(WRITE "${ROOT_PROJECT_PATH}/onnx_extended/_config.py" "${config_content}")
+message(STATUS "WRITE _CONFIG: '${SETUP_BUILD_LIB}/onnx_extended/_config.py'")
+file(WRITE "${SETUP_BUILD_LIB}/onnx_extended/_config.py" "${config_content}")
 
 #
 # test
diff --git a/_cmake/constants.cmake b/_cmake/constants.cmake
@@ -11,6 +11,7 @@ endif()
 message(STATUS "--------------------------------------------")
 message(STATUS "--------------------------------------------")
 message(STATUS "--------------------------------------------")
+message(STATUS "CMAKE_VERBOSE_MAKEFILE=${CMAKE_VERBOSE_MAKEFILE}")
 message(STATUS "ONNX_EXTENDED_VERSION=${ONNX_EXTENDED_VERSION}")
 message(STATUS "CMAKE_VERSION=${CMAKE_VERSION}")
 message(STATUS "CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}")
diff --git a/_cmake/externals/FindOrt.cmake b/_cmake/externals/FindOrt.cmake
@@ -95,30 +95,32 @@ endif()
 #
 # \arg:name target name
 # \arg:folder_copy where to copy the assembly
-# \arg:folder_copy_ort whenre to copy the dll for onnxruntime
 #
-function(ort_add_dependency name folder_copy folder_copy_ort)
+function(ort_add_dependency name folder_copy)
   get_target_property(target_output_directory ${name} BINARY_DIR)
   if(MSVC)
     set(destination_dir ${target_output_directory}/${CMAKE_BUILD_TYPE})
   else()
     set(destination_dir ${target_output_directory})
   endif()
-  message(STATUS "ort: copy ${ORT_LIB_FILES_LENGTH} files from '${ONNXRUNTIME_LIB_DIR}' "
-                 "to '${destination_dir}', '${folder_copy}', '${folder_copy_ort}' if it exists")
+  message(STATUS "ort: copy ${ORT_LIB_FILES_LENGTH} files from '${ONNXRUNTIME_LIB_DIR}'")
   foreach(file_i ${ORT_LIB_FILES})
+    message(STATUS "ort: copy ${file_i} to '${destination_dir}'")
     add_custom_command(
       TARGET ${name} POST_BUILD
       COMMAND ${CMAKE_COMMAND} ARGS -E copy ${file_i} ${destination_dir})
     if(folder_copy)
+      message(STATUS "ort: copy '${file_i}' to '${ROOT_PROJECT_PATH}/${folder_copy}'")
       add_custom_command(
         TARGET ${name} POST_BUILD
-        COMMAND ${CMAKE_COMMAND} ARGS -E copy ${file_i} ${folder_copy})
-    endif()
-    if(EXISTS folder_copy_ort)
+        COMMAND ${CMAKE_COMMAND} ARGS -E copy "${file_i}" "${ROOT_PROJECT_PATH}/${folder_copy}")
+      message(STATUS "ort: copy '${file_i}' to '${SETUP_BUILD_LIB}/${folder_copy}'")
+      add_custom_command(
+        TARGET ${name} POST_BUILD
+        COMMAND ${CMAKE_COMMAND} ARGS -E make_directory "${SETUP_BUILD_LIB}/${folder_copy}")
       add_custom_command(
         TARGET ${name} POST_BUILD
-        COMMAND ${CMAKE_COMMAND} ARGS -E copy ${file_i} ${folder_copy_ort})
+        COMMAND ${CMAKE_COMMAND} ARGS -E copy "${file_i}" "${SETUP_BUILD_LIB}/${folder_copy}")
     endif()
   endforeach()
   # file(COPY ${ORT_LIB_FILES} DESTINATION ${target_output_directory})
@@ -187,9 +189,17 @@ function(ort_add_custom_op name provider folder)
   endif()
   set_property(TARGET ${name} PROPERTY POSITION_INDEPENDENT_CODE ON)
   get_target_property(target_file ${name} LIBRARY_OUTPUT_NAME)
+  message(STATUS "ort: copy after build '$<TARGET_FILE:${name}>' to '${ROOT_PROJECT_PATH}/${folder}'")
+  add_custom_command(
+    TARGET ${name} POST_BUILD
+    COMMAND ${CMAKE_COMMAND} ARGS -E copy "$<TARGET_FILE:${name}>" "${ROOT_PROJECT_PATH}/${folder}")
+  message(STATUS "ort: copy after build '$<TARGET_FILE:${name}>' to '${SETUP_BUILD_LIB}//${folder}'")
+  add_custom_command(
+    TARGET ${name} POST_BUILD
+    COMMAND ${CMAKE_COMMAND} ARGS -E make_directory  "${SETUP_BUILD_LIB}/${folder}")
   add_custom_command(
     TARGET ${name} POST_BUILD
-    COMMAND ${CMAKE_COMMAND} ARGS -E copy $<TARGET_FILE:${name}> ${CMAKE_CURRENT_SOURCE_DIR}/${folder})
+    COMMAND ${CMAKE_COMMAND} ARGS -E copy "$<TARGET_FILE:${name}>" "${SETUP_BUILD_LIB}/${folder}")
 endfunction()
 
 include(FindPackageHandleStandardArgs)
diff --git a/_cmake/targets/ortinf.cmake b/_cmake/targets/ortinf.cmake
@@ -26,8 +26,7 @@ target_link_libraries(
 target_include_directories(ortinf PRIVATE ${ROOT_INCLUDE_PATH})
 ort_add_dependency(
   ortinf
-  ${CMAKE_CURRENT_SOURCE_DIR}/../onnx_extended/ortcy/wrap/
-  ${SETUP_BUILD_LIB}/onnx_extended/ortcy/wrap/)
+  onnx_extended/ortcy/wrap)
 
 set(ORTAPI_INCLUDE_DIR "${ROOT_INCLUDE_PATH}/onnx_extended/ortcy/wrap")
 
@@ -48,5 +47,5 @@ target_link_libraries(
   lib_ortapi
   onnxruntime
   common_kernels)
-ort_add_dependency(test_ortcy_inference_cpp "" "")
+ort_add_dependency(test_ortcy_inference_cpp "")
 add_test(NAME test_ortcy_inference_cpp COMMAND test_ortcy_inference_cpp)
diff --git a/_cmake/targets/ortops_optim_cpu.cmake b/_cmake/targets/ortops_optim_cpu.cmake
@@ -6,7 +6,7 @@ message(STATUS "+ KERNEL onnx_extended.ortops.optim.cpu")
 ort_add_custom_op(
   ortops_optim_cpu
   "CPU"
-  ../onnx_extended/ortops/optim/cpu
+  onnx_extended/ortops/optim/cpu
   ../onnx_extended/ortops/optim/cpu/tree_ensemble.cc
   ../onnx_extended/ortops/optim/cpu/ort_optim_cpu_lib.cc)
 
diff --git a/_cmake/targets/ortops_tutorial_cpu.cmake b/_cmake/targets/ortops_tutorial_cpu.cmake
@@ -6,7 +6,7 @@ message(STATUS "+ KERNEL onnx_extended.ortops.tutorial.cpu")
 ort_add_custom_op(
   ortops_tutorial_cpu
   "CPU"
-  ../onnx_extended/ortops/tutorial/cpu
+  onnx_extended/ortops/tutorial/cpu
   ../onnx_extended/ortops/tutorial/cpu/custom_gemm.cc
   ../onnx_extended/ortops/tutorial/cpu/dynamic_quantize_linear.cc
   ../onnx_extended/ortops/tutorial/cpu/my_kernel.cc
diff --git a/_cmake/targets/ortops_tutorial_cuda.cmake b/_cmake/targets/ortops_tutorial_cuda.cmake
@@ -9,7 +9,7 @@ if(CUDA_AVAILABLE)
   ort_add_custom_op(
     ortops_tutorial_cuda
     CUDA
-    ../onnx_extended/ortops/tutorial/cuda
+    onnx_extended/ortops/tutorial/cuda
     ../onnx_extended/ortops/tutorial/cuda/custom_gemm.cu
     ../onnx_extended/ortops/tutorial/cuda/ort_tutorial_cuda_lib.cc)
 
diff --git a/_doc/api/check.rst b/_doc/api/check.rst
@@ -3,6 +3,11 @@
 check
 =====
 
+check_installation
+==================
+
+.. autofunction:: onnx_extended.check_installation
+
 compiled_with_cuda
 ==================
 
diff --git a/onnx_extended/__init__.py b/onnx_extended/__init__.py
@@ -8,14 +8,44 @@
 __author__ = "Xavier Dupré"
 
 
-def check_installation():
+def check_installation(
+    ortops: bool = True, ortcy: bool = False, val: bool = True, verbose: bool = False
+):
     """
-    Checks the installation works.
+    Quickly checks the installation works.
+
+    :param ortops: checks that custom ops on CPU are working
+    :param ortcy: checks that OrtSession is working (cython bindings of onnxruntime)
+    :param val: checks that a couple of functions
+        in submodule validation are working
+    :param verbose: prints out which verifications is being processed
     """
     assert isinstance(get_cxx_flags(), str)
+    import datetime
     import warnings
 
+    def local_print(msg):
+        t = datetime.datetime.now().time()
+        print(msg.replace("[check_installation]", f"[check_installation] {t}"))
+
+    if val:
+        if verbose:
+            local_print("[check_installation] import numpy")
+        import numpy
+
+        if verbose:
+            local_print("[check_installation] import onnx-extended")
+        from onnx_extended.validation.cython.fp8 import cast_float32_to_e4m3fn
+
+        a = ((numpy.arange(10).astype(numpy.float32) - 5) / 10).astype(numpy.float32)
+        if verbose:
+            local_print("[check_installation] cast_float32_to_e4m3fn")
+        f8 = cast_float32_to_e4m3fn(a)
+        assert a.shape == f8.shape
+
     with warnings.catch_warnings(record=False):
+        if verbose:
+            local_print("[check_installation] import onnx, numpy")
         warnings.simplefilter("ignore")
         import numpy
         from onnx import TensorProto
@@ -27,8 +57,9 @@ def check_installation():
             make_opsetid,
         )
         from onnx.checker import check_model
-        from onnxruntime import InferenceSession, SessionOptions
-        from onnx_extended.ortops.tutorial.cpu import get_ort_ext_libs
+
+        if verbose:
+            local_print("[check_installation] create a simple onnx model")
 
         X = make_tensor_value_info("X", TensorProto.FLOAT, [None, None])
         A = make_tensor_value_info("A", TensorProto.FLOAT, [None, None])
@@ -43,18 +74,61 @@ def check_installation():
             ir_version=8,
         )
         check_model(onnx_model)
-
-        r = get_ort_ext_libs()
-        opts = SessionOptions()
-        opts.register_custom_ops_library(r[0])
-        sess = InferenceSession(
-            onnx_model.SerializeToString(), opts, providers=["CPUExecutionProvider"]
-        )
         a = numpy.random.randn(2, 2).astype(numpy.float32)
         b = numpy.random.randn(2, 2).astype(numpy.float32)
         feeds = {"X": a, "A": b}
-        got = sess.run(None, feeds)[0]
-        assert (a + b).shape == got.shape
+
+        if ortops:
+            if verbose:
+                local_print("[check_installation] import onnxruntime")
+            from onnxruntime import InferenceSession, SessionOptions
+
+            if verbose:
+                local_print("[check_installation] import onnx-extended")
+            from onnx_extended.ortops.tutorial.cpu import get_ort_ext_libs
+
+            r = get_ort_ext_libs()
+            if verbose:
+                local_print(
+                    f"[check_installation] get_ort_ext_libs()={get_ort_ext_libs()!r}"
+                )
+            opts = SessionOptions()
+            opts.register_custom_ops_library(r[0])
+            if verbose:
+                local_print("[check_installation] create session")
+            sess = InferenceSession(
+                onnx_model.SerializeToString(), opts, providers=["CPUExecutionProvider"]
+            )
+            if verbose:
+                local_print("[check_installation] run session")
+            got = sess.run(None, feeds)[0]
+            if verbose:
+                local_print("[check_installation] check shapes")
+            assert (a + b).shape == got.shape
+
+        if ortcy:
+            if verbose:
+                local_print("[check_installation] import onnx-extended")
+            from onnx_extended.ortops.tutorial.cpu import get_ort_ext_libs
+            from onnx_extended.ortcy.wrap.ortinf import OrtSession
+
+            r = get_ort_ext_libs()
+            if verbose:
+                local_print(
+                    f"[check_installation] get_ort_ext_libs()={get_ort_ext_libs()!r}"
+                )
+            if verbose:
+                local_print("[check_installation] create OrtSession")
+            session = OrtSession(onnx_model.SerializeToString(), custom_libs=r)
+            if verbose:
+                local_print("[check_installation] run OrtSession")
+            got = session.run([a, b])
+            if verbose:
+                local_print("[check_installation] check shapes")
+            assert (a + b).shape == got[0].shape
+
+        if verbose:
+            local_print("[check_installation] done")
 
 
 def has_cuda() -> bool:
diff --git a/onnx_extended/_command_lines_parser.py b/onnx_extended/_command_lines_parser.py
diff --git a/pyproject.toml b/pyproject.toml
diff --git a/setup.py b/setup.py

Original file line number	Diff line number	Diff line change
`@@ -86,9 +86,11 @@ include("finalize.cmake")`
`86`	`86`	`#`
`87`	`87`
`88`	`88`	`message(STATUS "---------------------------------------------------")`
`89`		`-message(STATUS "WRITE _CONFIG: '${ROOT_PROJECT_PATH}/onnx_extended/_config.py'")`
`90`	`89`	`message(STATUS "CONTENT\n ${config_content}")`
	`90`	`+message(STATUS "WRITE _CONFIG: '${ROOT_PROJECT_PATH}/onnx_extended/_config.py'")`
`91`	`91`	`file(WRITE "${ROOT_PROJECT_PATH}/onnx_extended/_config.py" "${config_content}")`
	`92`	`+message(STATUS "WRITE _CONFIG: '${SETUP_BUILD_LIB}/onnx_extended/_config.py'")`
	`93`	`+file(WRITE "${SETUP_BUILD_LIB}/onnx_extended/_config.py" "${config_content}")`
`92`	`94`
`93`	`95`	`#`
`94`	`96`	`# test`