Release management: Move all requirements; Fix rucio#4456

stfc · Jun 16, 2021 · 4c4deb1 · 4c4deb1
1 parent d1b972d
commit 4c4deb1
Show file tree

Hide file tree

Showing 20 changed files with 390 additions and 656 deletions.
diff --git a/MANIFEST.in.client b/MANIFEST.in.client
@@ -3,16 +3,14 @@ include AUTHORS
 include LICENSE
 include ChangeLog
 include README.rst
+include setuputil.py
 include setup.py
 include setup.cfg
+include pyproject.toml
+include requirements.txt
 include etc/rse-accounts.cfg.template
 include etc/rucio.cfg.atlas.client.template
 include etc/rucio.cfg.template
-include etc/pip-requires-client
-include etc/rse-accounts.cfg.template
-include etc/rucio.cfg.atlas.client.template
-include etc/rucio.cfg.template
-include tools/pip-requires-client
 recursive-include  etc/schemas *.json
 prune dist
 prune examples

diff --git a/MANIFEST.in.rucio b/MANIFEST.in.rucio
@@ -3,15 +3,15 @@ include AUTHORS
 include LICENSE
 include ChangeLog
 include README.rst
+include setuputil.py
 include setup.py
 include setup.cfg
+include pyproject.toml
+include requirements.txt
 include lib/rucio/db/migrate_repo/script.py.mako
 include etc/rse-accounts.cfg.template
 include etc/rucio.cfg.atlas.client.template
 include etc/rucio.cfg.template
-include etc/pip-requires
-include etc/pip-requires-test
-include etc/pip-requires-client
 include tools/bootstrap.py
 include tools/reset_database.py
 recursive-include lib/rucio/db/sqla/migrate_repo/versions *.py

diff --git a/MANIFEST.in.webui b/MANIFEST.in.webui
@@ -3,8 +3,11 @@ include AUTHORS
 include LICENSE
 include ChangeLog
 include README.rst
+include setuputil.py
 include setup.py
 include setup.cfg
+include pyproject.toml
+include requirements.txt
 recursive-include lib/rucio/web/ui *
 prune dist
 prune examples

diff --git a/etc/docker/test/centos7.Dockerfile b/etc/docker/test/centos7.Dockerfile
@@ -118,6 +118,18 @@ RUN curl https://www.sqlite.org/2019/sqlite-autoconf-3290000.tar.gz | tar xzv &&
 
 WORKDIR /usr/local/src/rucio
 
+COPY requirements.txt setuputil.py ./
+
+# pre-install requirements
+RUN if [[ "$PYTHON" == "2.7" ]] ; then \
+        python3 -m pip --no-cache-dir install --upgrade -r requirements.txt && \
+        python3 -c "from setuputil import *; clients_requirements_table.update({'dev': dev_requirements}); list_all_requirements(clients_requirements_table)" > py2_requirements.txt && \
+        python2 -m pip --no-cache-dir install --upgrade -r py2_requirements.txt && \
+        python2 -m pip list ; \
+    else \
+        python -m pip --no-cache-dir install --upgrade -r requirements.txt ; \
+    fi
+
 COPY etc etc
 
 RUN mkdir -p /var/log/rucio/trace && \
@@ -137,15 +149,6 @@ RUN rpm -i etc/docker/test/extra/oic.rpm; \
     echo "/usr/lib/oracle/12.2/client64/lib" >/etc/ld.so.conf.d/oracle.conf; \
     ldconfig
 
-# pre-install requirements
-RUN if [ "$PYTHON" == "2.7" ] ; then \
-        python3 -m pip --no-cache-dir install --upgrade -r etc/pip-requires && \
-        python2 -m pip --no-cache-dir install --upgrade -r etc/pip-requires-client -r etc/pip-requires-test && \
-        python2 -m pip list ; \
-    else \
-        python -m pip --no-cache-dir install --upgrade -r etc/pip-requires -r etc/pip-requires-client -r etc/pip-requires-test ; \
-    fi
-
 # copy everything else except the git-dir (anything above is cache-friendly)
 COPY .flake8 .pep8 .pycodestyle pylintrc setup.py setup_rucio.py setup_rucio_client.py setup_webui.py ./
 COPY tools tools
@@ -154,7 +157,7 @@ COPY lib lib
 
 # Install Rucio server + dependencies
 RUN if [ "$PYTHON" == "2.7" ] ; then PYEXEC=python3 ; else PYEXEC=python ; fi ; \
-    $PYEXEC -m pip --no-cache-dir install --upgrade .[oracle,postgresql,mysql,kerberos,dev,saml] && \
+    $PYEXEC -m pip --no-cache-dir install --upgrade .[oracle,postgresql,mysql,kerberos,saml,dev] && \
     $PYEXEC -m pip list
 
 WORKDIR /opt/rucio

diff --git a/etc/docker/test/fedora32.Dockerfile b/etc/docker/test/fedora32.Dockerfile
@@ -26,10 +26,17 @@ RUN test "x${PYTHON}" = "x3.8" && \
     alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 && \
     python -m pip --no-cache-dir install --upgrade pip && \
     python -m pip --no-cache-dir install --upgrade setuptools wheel && \
-    dnf clean all
+    dnf clean all && \
+    rpm -i https://yum.oracle.com/repo/OracleLinux/OL8/oracle/instantclient21/x86_64/getPackage/oracle-instantclient-basiclite-21.1.0.0.0-1.x86_64.rpm && \
+    echo "/usr/lib/oracle/21/client64/lib" > /etc/ld.so.conf.d/oracle-instantclient.conf && \
+    ldconfig
 
 WORKDIR /usr/local/src/rucio
 
+# pre-install requirements
+COPY requirements.txt requirements.txt
+RUN python -m pip --no-cache-dir install --upgrade -r requirements.txt
+
 COPY etc etc
 
 RUN mkdir -p /var/log/rucio/trace && \
@@ -45,15 +52,8 @@ RUN mkdir -p /var/log/rucio/trace && \
     cp etc/certs/ruciouser.key.pem etc/ruciouser.key.pem && \
     chmod 0400 etc/ruciouser.key.pem
 
-RUN rpm -i https://yum.oracle.com/repo/OracleLinux/OL8/oracle/instantclient21/x86_64/getPackage/oracle-instantclient-basiclite-21.1.0.0.0-1.x86_64.rpm && \
-    echo "/usr/lib/oracle/21/client64/lib" > /etc/ld.so.conf.d/oracle-instantclient.conf && \
-    ldconfig
-
-# pre-install requirements
-RUN python -m pip --no-cache-dir install --upgrade -r etc/pip-requires -r etc/pip-requires-client -r etc/pip-requires-test
-
 # copy everything else except the git-dir (anything above is cache-friendly)
-COPY .flake8 .pep8 .pycodestyle pylintrc setup.py setup_rucio.py setup_rucio_client.py setup_webui.py ./
+COPY .flake8 .pep8 .pycodestyle pylintrc setuputil.py setup.py setup_rucio.py setup_rucio_client.py setup_webui.py ./
 COPY tools tools
 COPY bin bin
 COPY lib lib

diff --git a/etc/docker/test/fedora33.Dockerfile b/etc/docker/test/fedora33.Dockerfile
@@ -26,10 +26,17 @@ RUN test "x${PYTHON}" = "x3.9" && \
     alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 && \
     python -m pip --no-cache-dir install --upgrade pip && \
     python -m pip --no-cache-dir install --upgrade setuptools wheel && \
-    dnf clean all
+    dnf clean all && \
+    rpm -i https://yum.oracle.com/repo/OracleLinux/OL8/oracle/instantclient21/x86_64/getPackage/oracle-instantclient-basiclite-21.1.0.0.0-1.x86_64.rpm && \
+    echo "/usr/lib/oracle/21/client64/lib" > /etc/ld.so.conf.d/oracle-instantclient.conf && \
+    ldconfig
 
 WORKDIR /usr/local/src/rucio
 
+# pre-install requirements
+COPY requirements.txt requirements.txt
+RUN python -m pip --no-cache-dir install --upgrade -r requirements.txt
+
 COPY etc etc
 
 RUN mkdir -p /var/log/rucio/trace && \
@@ -45,15 +52,8 @@ RUN mkdir -p /var/log/rucio/trace && \
     cp etc/certs/ruciouser.key.pem etc/ruciouser.key.pem && \
     chmod 0400 etc/ruciouser.key.pem
 
-RUN rpm -i https://yum.oracle.com/repo/OracleLinux/OL8/oracle/instantclient21/x86_64/getPackage/oracle-instantclient-basiclite-21.1.0.0.0-1.x86_64.rpm && \
-    echo "/usr/lib/oracle/21/client64/lib" > /etc/ld.so.conf.d/oracle-instantclient.conf && \
-    ldconfig
-
-# pre-install requirements
-RUN python -m pip --no-cache-dir install --upgrade -r etc/pip-requires -r etc/pip-requires-client -r etc/pip-requires-test
-
 # copy everything else except the git-dir (anything above is cache-friendly)
-COPY .flake8 .pep8 .pycodestyle pylintrc setup.py setup_rucio.py setup_rucio_client.py setup_webui.py ./
+COPY .flake8 .pep8 .pycodestyle pylintrc setuputil.py setup.py setup_rucio.py setup_rucio_client.py setup_webui.py ./
 COPY tools tools
 COPY bin bin
 COPY lib lib

diff --git a/etc/pip-requires b/etc/pip-requires
diff --git a/etc/pip-requires-client b/etc/pip-requires-client
diff --git a/etc/pip-requires-test b/etc/pip-requires-test
diff --git a/lib/rucio/tests/test_heartbeat.py b/lib/rucio/tests/test_heartbeat.py
@@ -29,16 +29,16 @@
 import pytest
 
 from rucio.core.heartbeat import live, die, cardiac_arrest, list_payload_counts, list_heartbeats, sanity_check
-from rucio.db.sqla.session import transactional_session
 from rucio.db.sqla.models import Heartbeats
+from rucio.db.sqla.session import transactional_session
 
 
 @pytest.mark.dirty
 @pytest.mark.noparallel(reason='using the same executable for multiple tests')
 class TestHeartbeat(unittest.TestCase):
 
     def __pid(self):
-        return random.randint(0, 2**16)
+        return random.randint(2, 2**16)
 
     def __thread(self):
         thread = threading.Thread()

diff --git a/lib/rucio/web/ui/flask/common/utils.py b/lib/rucio/web/ui/flask/common/utils.py
@@ -238,7 +238,7 @@ def validate_webui_token(from_cookie=True, session_token=None):
     if not valid_token_dict or not session_token:
         return None
     else:
-        valid_token_dict['token'] = session_token
+        valid_token_dict['token'] = session_token  # pylint: disable=E1137
         return valid_token_dict
 
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,67 @@
+# All dependencies needed to run rucio client (and server/daemons) should be defined here
+requests>=2.20.0,<=2.25.0                                   # Python HTTP for Humans.
+urllib3>=1.24.2,<=1.26.4                                    # HTTP library with thread-safe connection pooling, file post, etc.
+dogpile.cache>=0.6.5,<0.7.0; python_version < '3.0'         # Caching API plugins (0.7.0 only Py3 compatible)
+dogpile.cache>=0.6.5,<=1.1.1; python_version >= '3.0'       # Caching API plugins
+tabulate>=0.8.0,<0.9.0                                      # Pretty-print tabular data
+six>=1.12.0<1.16.0                                          # Python 2 and 3 compatibility utilities
+jsonschema>=3.2.0                                           # For JSON schema validation (Policy modules)
+
+# All dependencies needed in extras for rucio client (and server/daemons) should be defined here
+paramiko==2.7.2                                             # ssh_extras; SSH2 protocol library (also needed in the server)
+kerberos>=1.3.0                                             # kerberos_extras for client and server
+pykerberos>=1.2.1                                           # kerberos_extras for client and server
+requests-kerberos>=0.12.0                                   # kerberos_extras for client and server
+python-swiftclient>=3.5.0                                   # swift_extras
+argcomplete>=1.9.0,<=1.12.2                                 # argcomplete_extras; Bash tab completion for argparse
+boto>=2.49.0,<2.50.0                                        # s3_extras; S3 boto protocol
+pysftp>=0.2.9,<0.3                                          # sftp_extras; Used by sftp protocol
+bz2file>=0.98,<0.99                                         # dumper_extras; Read and write bzip2-compressed files.
+python-magic>=0.4.15,<0.5.0                                 # dumper_extras; File type identification using libmagic
+#gfal2-python                                                # dumper_extras and gfal2_extras ~~ BROKEN in pypi
+
+# All dependencies needed to run rucio server/daemons should be defined here
+SQLAlchemy==1.3.17                                          # DB backend -- cannot move to 1.3.18-1.3.20 due to MySQL5 NOWAIT bug in SQLAlchemy
+alembic==1.4.3                                              # Lightweight database migration tool for SQLAlchemy
+python-memcached==1.59                                      # Quick and small memcached client for Python3
+python-dateutil==2.8.1                                      # Extensions to the standard datetime module
+stomp.py==6.1.1                                             # ActiveMQ Messaging Protocol
+statsd==3.3.0                                               # Needed to log into graphite with more than 1 Hz
+geoip2==4.1.0                                               # GeoIP2 API (for IPv6 support)
+oauth2client==4.1.3                                         # OAuth 2.0 client library
+retrying==1.3.3                                             # general-purpose retrying library to simplify the task of adding retry behavior to just about anything
+redis==3.5.3                                                # Python client for Redis key-value store
+numpy==1.19.4                                               # Numpy for forecasting T3C
+Flask==1.1.2                                                # Python web framework
+oic==1.2.1                                                  # for authentication via OpenID Connect protocol
+prometheus_client==0.9.0                                    # Python client for the Prometheus monitoring system
+boto3>=1.9.130,<1.17.0                                      # S3 boto protocol (new version)
+
+# All dependencies needed in extras for rucio server/daemons should be defined here
+cx_oracle==8.0.1                                            # oracle_extras
+psycopg2-binary==2.8.6                                      # postgresql_extras
+PyMySQL                                                     # mysql_extras
+PyYAML==5.4                                                 # globus_extras and used for reading test configuration files
+globus-sdk==1.8.0                                           # globus_extras
+python3-saml>=1.6.0                                         # saml_extras
+
+# All dependencies needed to develop/test rucio should be defined here
+pytest==4.6.11; python_version < '3.6'                      # Used for running tests
+pytest==6.1.2; python_version >= '3.6'
+pytest-xdist~=2.2.0; python_version >= '3.6'                # Used for parallel testing on Python 3.6+
+pyflakes==2.3.1                                             # Passive checker of Python programs
+flake8==3.9.1                                               # Wrapper around PyFlakes&pep8
+pylint==1.9.4; python_version < '3.6'                       # static code analysis. 1.9.5 last 2.7 compatible release
+pylint==2.8.2; python_version >= '3.6'
+astroid<2.5.7                                               # things break in 2.5.7 and everyone is to blame
+isort>=4.2.5,<5                                             # pylint up to now (2.5.3) does not support isort 5
+virtualenv==20.2.1                                          # Virtual Python Environment builder
+xmltodict==0.12.0                                           # Makes working with XML feel like you are working with JSON
+pytz==2020.4                                                # World timezone definitions, modern and historical
+subprocess32==3.5.4; python_version <= '3.0'                # A backport of the subprocess module from Python 3.2/3.3 for use on 2.x.
+pycodestyle==2.7.0                                          # New package replacing pep8
+mock==3.0.5; python_version < '3.3'                         # Mock package needed for readthedocs build for mocking pycurls (mock seems to break SQLAlchemy in Python 3 tests + since Python 3.3 there is unittest.mock)
+s3cmd==2.0.2                                                # Used for testing of S3
+pydoc-markdown~=3.11.0; python_version >= '3.5'             # Used for generating Markdown documentation for docusaurus
+docspec_python==0.1.0; python_version >= '3.5'              # FIX incompatibility with pydoc-markdown
+sh~=1.14.1                                                  # Convenience library for running subprocesses in Python