Python MapScript updates (MapServer#7221)

* Switch to python3-venv and remove virtualenv requirement

* Remove Python2 constraints

* Hide pip upgrade warning

* Switch to demo.mapserver.org for test

* Ensure CMake stamp files are created

* Set CURL_CA_BUNDLE for Windows CI

* Ensure location of stamp files

* Add --no-warn-script-location

* Remove warnings from CI msautotests

.github/workflows/start.sh

@@ -9,7 +9,7 @@ export PYTHON_VERSION=system
 LANG=en_US.UTF-8
 export LANG
 DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-    sudo locales tzdata software-properties-common python3-dev python3-pip python3-setuptools git curl \
+    sudo locales tzdata software-properties-common python3-dev python3-venv python3-pip python3-setuptools git curl \
     apt-transport-https ca-certificates gnupg software-properties-common wget
 #install PHP 8.1
 DEBIAN_FRONTEND=noninteractive add-apt-repository ppa:ondrej/php -y

appveyor.yml

@@ -91,6 +91,7 @@ build_script:
   # set the MapScript custom environment variable for python 3.8+
   - set MAPSERVER_DLL_PATH=%BUILD_FOLDER%/build/Release;%SDK_BIN%
   - set PROJ_DATA=%SDK_BIN%/proj9/SHARE
+  - set CURL_CA_BUNDLE=%SDK_BIN%/curl-ca-bundle.crt
   # ensure the GIF driver is available for tests
   - set GDAL_DRIVER_PATH=%SDK_BIN%/gdal/plugins
   # check the mapserver exe can run
@@ -100,8 +101,9 @@ build_script:
 before_test:
   - set PATH=%SDK_BIN%/gdal/apps;%PATH%;
   - cd %BUILD_FOLDER%/msautotest
+  - set PIP_NO_PYTHON_VERSION_WARNING=1
   - "./mssql/create_mssql_db.bat"
-  - "%Python_ROOT_DIR%/python -m pip install -U -r requirements.txt"
+  - "%Python_ROOT_DIR%/python -m pip install --no-warn-script-location -U -r requirements.txt"
   - "%Python_ROOT_DIR%/python -m pip install --no-index --find-links=file://%BUILD_FOLDER%/build/src/mapscript/python/Release/dist mapscript"
 
 test_script:

msautotest/requirements.txt

@@ -1,10 +1,4 @@
-# Python 2.7 constraints
-# more-itertools 6.0 no longer supporting Python 2.7
-more-itertools < 6.0.0; python_version < '3.0'
-# pytest 5.0 no longer supporting Python 2.7
-pytest>=3.6.0,<5.0.0; python_version < '3.0'
-
-pytest>=3.6.0; python_version >= '3.0'
+pytest
 pytest-sugar
 pytest-env
 lxml

scripts/vagrant/packages.sh

@@ -12,7 +12,7 @@ apt-get -y upgrade
 
 # install packages we need
 apt-get install -q -y git build-essential pkg-config cmake libgeos-dev rake \
-    libpq-dev python3-dev python3-pip libproj-dev libxml2-dev postgis php-dev \
+    libpq-dev python3-dev python3-pip python3-venv libproj-dev libxml2-dev postgis php-dev \
     postgresql-server-dev-12 postgresql-12-postgis-3 postgresql-12-postgis-3-scripts vim bison flex swig \
     librsvg2-dev libpng-dev libjpeg-dev libgif-dev \
     libfreetype6-dev libfcgi-dev libcurl4-gnutls-dev libcairo2-dev \

src/mapscript/python/CMakeLists.txt

@@ -60,32 +60,33 @@ endif()
 
 add_custom_target(
     pythonmapscript-wheel
-    DEPENDS mapscripttests.stamp
+    DEPENDS ${MAPSCRIPT_WORKING_DIR}/mapscripttests.stamp
 )
 
 add_custom_command(
     DEPENDS ${SWIG_MODULE_pythonmapscript_REAL_NAME}
-    OUTPUT mapscriptvenv.stamp
+    OUTPUT ${MAPSCRIPT_WORKING_DIR}/mapscriptvenv.stamp
     WORKING_DIRECTORY ${MAPSCRIPT_WORKING_DIR}
-    COMMAND ${Python_EXECUTABLE} -m pip install pip --upgrade
-    COMMAND ${Python_EXECUTABLE} -m pip install virtualenv
-    COMMAND ${Python_EXECUTABLE} -m virtualenv mapscriptvenv
-    COMMAND ${Python_VENV_SCRIPTS}/pip install -r ${PROJECT_SOURCE_DIR}/src/mapscript/python/requirements-dev.txt > requires.log
+    COMMAND ${Python_EXECUTABLE} -m venv mapscriptvenv
+    COMMAND ${CMAKE_COMMAND} -E env PIP_NO_PYTHON_VERSION_WARNING=1
+            ${Python_VENV_SCRIPTS}/pip install --no-warn-script-location  -r ${PROJECT_SOURCE_DIR}/src/mapscript/python/requirements-dev.txt > requires.log
+    COMMAND ${CMAKE_COMMAND} -E touch ${MAPSCRIPT_WORKING_DIR}/mapscriptvenv.stamp
     COMMENT "Creating a Python virtual environment and installing the required packages"
 )
 
 add_custom_command(
-    DEPENDS mapscriptvenv.stamp
-    OUTPUT mapscriptwheel.stamp
+    DEPENDS ${MAPSCRIPT_WORKING_DIR}/mapscriptvenv.stamp
+    OUTPUT ${MAPSCRIPT_WORKING_DIR}/mapscriptwheel.stamp
     WORKING_DIRECTORY ${MAPSCRIPT_WORKING_DIR}
     COMMAND ${Python_VENV_SCRIPTS}/python -m build --wheel > wheel_build.log
+    COMMAND ${CMAKE_COMMAND} -E touch ${MAPSCRIPT_WORKING_DIR}/mapscriptwheel.stamp
     COMMENT "Building the mapscript Python wheel"
 )
 
 add_custom_command(
     WORKING_DIRECTORY ${Python_VENV_SCRIPTS} # make sure scripts aren't run when from the same folder as mapscript.py
-    DEPENDS mapscriptwheel.stamp
-    OUTPUT mapscripttests.stamp
+    DEPENDS ${MAPSCRIPT_WORKING_DIR}/mapscriptwheel.stamp
+    OUTPUT ${MAPSCRIPT_WORKING_DIR}/mapscripttests.stamp
     COMMAND ${Python_VENV_SCRIPTS}/pip install --no-index --find-links=${MAPSCRIPT_WORKING_DIR}/dist mapscript
     # ERROR: file or package not found: mapscript.tests (missing __init__.py?) is caused by
     # ImportError: DLL load failed while importing _mapscript: The specified module could not be found.
@@ -94,6 +95,7 @@ add_custom_command(
     COMMAND ${Python_VENV_SCRIPTS}/python -m mapscript.examples.shpdump ${PROJECT_SOURCE_DIR}/tests/polygon.shp > shpdump.txt
     COMMAND ${Python_VENV_SCRIPTS}/python -m mapscript.examples.shpinfo ${PROJECT_SOURCE_DIR}/tests/polygon.shp > shpinfo.txt
     COMMAND ${Python_VENV_SCRIPTS}/python -m mapscript.examples.wxs ${PROJECT_SOURCE_DIR}/tests/test.map > response.xml
+    COMMAND ${CMAKE_COMMAND} -E touch ${MAPSCRIPT_WORKING_DIR}/mapscripttests.stamp
     COMMENT "Installing the Python wheel and testing it in the virtual environment, then running tests and examples"
 )
 

src/mapscript/python/requirements-dev.txt

@@ -2,4 +2,4 @@ pytest
 pillow
 wheel>=0.38.0
 setuptools>=45.0.0
-build[virtualenv]
+build

src/mapscript/python/tests/cases/thread_test.py

@@ -148,16 +148,15 @@ def draw_map_wms(name, save=0):
 
     # print("making map in thread %s" % (name))
     mo = mapscript.mapObj(TESTMAPFILE)
-    # WFS layer
+    # WMS layer
     lo = mapscript.layerObj()
-    lo.name = "jpl_wms"
+    lo.name = "world_latlong"
     lo.setProjection("+init=epsg:4326")
     lo.connectiontype = mapscript.MS_WMS
-    lo.connection = "http://vmap0.tiles.osgeo.org/wms/vmap0?"
+    lo.connection = "https://demo.mapserver.org/cgi-bin/msautotest?"
     lo.metadata.set("wms_service", "WMS")
-    lo.metadata.set("wms_server_version", "1.1.1")
-    lo.metadata.set("wms_name", "basic")
-    lo.metadata.set("wms_style", "visual")
+    lo.metadata.set("wms_server_version", "1.3.0")
+    lo.metadata.set("wms_name", "world_latlong")
     lo.metadata.set("wms_format", "image/jpeg")
     lo.type = mapscript.MS_LAYER_RASTER
     lo.status = mapscript.MS_DEFAULT