From 614ad535cddf08fb568933e5f3174b3788f95c26 Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea <ssbarnea@redhat.com> Date: Tue, 3 Apr 2018 19:10:06 +0100 Subject: [PATCH] build cleanup - caching - adop stages - auto-release on tag - switched to hacking linting --- .gitignore | 1 + .pep8 | 17 ----- .travis.yml | 149 ++++++++++++++++++++----------------------- demo/demo_colorer.py | 2 - docs/conf.py | 11 ++-- requirements-dev.txt | 5 +- setup.cfg | 39 +++-------- tendo/__init__.py | 16 +++-- tendo/ansiterm.py | 4 +- tendo/colorer.py | 6 +- tendo/execfile2.py | 23 ++++--- tendo/singleton.py | 13 ++-- tendo/tee.py | 20 +++--- tendo/unicode.py | 17 ++--- tendo/version.py | 3 - tox.ini | 33 ++++++---- 16 files changed, 153 insertions(+), 206 deletions(-) delete mode 100644 .pep8 delete mode 100755 tendo/version.py diff --git a/.gitignore b/.gitignore index b0d89a3..ec59132 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ test-distribute.sh .idea/tendo.iml /ChangeLog /AUTHORS +/.pytest_cache diff --git a/.pep8 b/.pep8 deleted file mode 100644 index f4268a5..0000000 --- a/.pep8 +++ /dev/null @@ -1,17 +0,0 @@ -[pep8] -exclude=lib,.tox,third,*.egg,docs -;filename= -;select -ignore=E501 scripts -max-line-length=1024 -count=1 -;format -;quiet -;show-pep8 -;show-source -statistics=1 -;verbose=1 - -;PEP8_OPTS="--filename=*.py --exclude=lib --ignore=E501 scripts" -;pep8 $PEP8_OPTS --show-source --repeat -;pep8 --statistics -qq $PEP8_OPTS \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index fd5643b..acbf342 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,88 +1,77 @@ language: python +cache: +- pip sudo: false -matrix: - fast_finish: false os: - linux -matrix: - include: - - python: 2.7 - env: TOXENV=py27 - - python: 3.4 - env: TOXENV=py34 - - python: 3.5 - env: TOXENV=py35 - - python: 3.6 - env: TOXENV=py36 - - python: pypy - env: TOXENV=pypy - - python: 2.7 - env: TOXENV=docs -# https://docs.travis-ci.com/user/customizing-the-build -branches: - only: - - master - - develop - - /^\d+\.\d+(\.\d+)?(-\S*)?$/ +stages: +- lint +- docs +- test +- deploy install: -- pip -q --log dist/pip.log install --upgrade pip setuptools tox-travis py wheel -- python setup.py sdist bdist_wheel install -- pip install ./dist/*.whl -- pip --version -script: -- export PACKAGE_NAME=$(python setup.py --name) -- export PACKAGE_VERSION=$(python setup.py --version) -- python setup.py --version -- tox --installpkg ./dist/*.whl --travis-after -# validates that the build source distribution is installable using the old easy_install -- pip uninstall -y $PACKAGE_NAME && easy_install ./dist/$PACKAGE_NAME-*.tar.gz -after_success: -- coveralls -- bash <(curl -s https://codecov.io/bash) -- requires.io update-site -t ac3bbcca32ae03237a6aae2b02eb9411045489bb -r -notifications: - email: - - pycontribs@googlegroups.com - - sorin.sbarnea@gmail.com - -deploy: -- provider: releases - api_key: - secure: FdDsSx1GhjAcvNPtdDhNtlIlAVFDWv+hXu1HGLnDVdAzFVnJt96z7ORTjKuUnTDNPKvyxOtAx67xE5K2pR9UV8MK4rz3qyRNoNbUKz9IHqs8BCdm7NTUV1GgoTxlFvbez5K/B8IdEge37ru+qE5zJslQYSBAxA1y+lUaNAhugiA= - file_glob: true - file: - - dist/$PACKAGE_NAME-$PACKAGE_VERSION* - - ChangeLog - skip_cleanup: true - on: - repo: pycontribs/tendo - tags: true +- pip install -q tox-travis tox-pyenv virtualenv wheel | cat +jobs: + include: + - stage: lint + script: python -m tox python: 2.7 - condition: $TOXENV == py27 -- provider: pypi - user: pycontribs - password: - secure: "ZvNpERuRpiPb5+mML/nm16UTI3DwEZEVOLmz9kDfaJU5+9hy1mvhwkIBWEEEBvy3NA5zk64i1uBORSwp+WPnwWNsurksZwgmSp1k2M6FzE0l2tRt7VQ4Lle6CfhLJ7vo0mZS7GdfWwcw4DEOV7guh3VtD200TNK6o/AuRRXztpM=" - distributions: sdist bdist_wheel - skip_cleanup: true - on: - tags: true + env: TOXENV=lint + - stage: docs + script: python -m tox python: 2.7 - condition: $TOXENV == py27 - branch: master -- provider: pypi - server: https://testpypi.python.org/pypi - user: pycontribs - password: - secure: "ZvNpERuRpiPb5+mML/nm16UTI3DwEZEVOLmz9kDfaJU5+9hy1mvhwkIBWEEEBvy3NA5zk64i1uBORSwp+WPnwWNsurksZwgmSp1k2M6FzE0l2tRt7VQ4Lle6CfhLJ7vo0mZS7GdfWwcw4DEOV7guh3VtD200TNK6o/AuRRXztpM=" - distributions: sdist bdist_wheel - skip_cleanup: true - on: - tags: false + env: TOXENV=docs + - stage: test + script: python -m tox python: 2.7 - condition: $TOXENV == py27 - branch: develop -env: - global: - - secure: fuXwQL+KHQ96XkAFl2uQc8eK8dAjrgkup46tck/UGjVpdv1PT/yHmBKrvpFjDa50ueGbtBwTdKAwhyAmYuiZCk2IYHzdvBylCZBBji2FSpaTM59CVwgkVT6tx3HHO83X0mEX6ih9TJvZD5XhX+YUjopnseRXRq3ey3JZJXWN4RM= - - secure: "pGQGM5YmHvOgaKihOyzb3k6bdqLQnZQ2OXO9QrfXlXwtop3zvZQi80Q+01l230x2psDWlwvqWTknAjAt1w463fYXPwpoSvKVCsLSSbjrf2l56nrDqnoir+n0CBy288+eIdaGEfzcxDiuULeKjlg08zrqjcjLjW0bDbBrlTXsb5U=" + env: TOXENV=py27 + after_success: + - coveralls + - bash <(curl -s https://codecov.io/bash) -e TOX_ENV + - stage: test + script: python -m tox + python: 3.4 + env: TOXENV=py34 + after_success: + - coveralls + - bash <(curl -s https://codecov.io/bash) -e TOX_ENV + - stage: test + script: python -m tox + python: 3.5 + env: TOXENV=py35 + after_success: + - coveralls + - bash <(curl -s https://codecov.io/bash) -e TOX_ENV + - stage: test + script: python -m tox + python: 3.6 + env: TOXENV=py36 PYTHON='3.6' PYENV_VERSION='system' + after_success: + - coveralls + - bash <(curl -s https://codecov.io/bash) -e TOX_ENV + - stage: deploy + script: + - export PACKAGE_NAME=$(python setup.py --name) + - export PACKAGE_VERSION=$(python setup.py --version) + - python setup.py sdist bdist_wheel + deploy: + - provider: pypi + user: pycontribs + password: + secure: beq1+egB93Y9a51NOdrG0ja9zKb+g3/JejoM6kUpm2FU37a9+AU3fAAoGdYcrRqF2fmrBGuYNW29vJEzaDT4YK/FSMNrcipt8U3yyNXJ0oFbVsi0FLeDlBzEFarhY71BguEoJlf+nwCphPXCj/aJPxNiAKYnNCfBdfP0WKH/N5M= + distributions: sdist bdist_wheel + skip_cleanup: true + on: + tags: true + repo: pycontribs/tendo + - provider: releases + api_key: + secure: Dz7yMcIBxTKD2zYi0ph9qTz2N0AdmOWD9eI/CMW3DdXh15e4fQqL7O5cr4Sn+FSKHMbUCUoztXwJsYLbrCPB2tH8HliPhSuNzb0CwScRD9wSzQzhli10YPj+Oe2UlPOcIWQFRYTUQ1tfpvfDcvRL7klLDiKUpghJ+xf7crXzqGE= + file: + - dist/$PACKAGE_NAME-$PACKAGE_VERSION.tar.gz + - dist/$PACKAGE_NAME-$PACKAGE_VERSION-py2.py3-none-any.whl + - ChangeLog + skip_cleanup: true + on: + tags: true + repo: pycontribs/tendo diff --git a/demo/demo_colorer.py b/demo/demo_colorer.py index df0bcf3..75bab59 100755 --- a/demo/demo_colorer.py +++ b/demo/demo_colorer.py @@ -1,7 +1,5 @@ #!/usr/bin/env python # encoding: utf-8 -# Author: sorin sbarnea -# License: public domain from tendo import colorer # noqa diff --git a/docs/conf.py b/docs/conf.py index e9b3a04..8d9230c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,14 +11,13 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys -import os import inspect +import os +import sys cmd_folder = os.path.realpath(os.path.join(os.path.abspath(os.path.split(inspect.getfile(inspect.currentframe()))[0]), "..")) if cmd_folder not in sys.path: sys.path.insert(0, cmd_folder) -from tendo import version as tendo_version # noqa - +from tendo import __version__ # noqa:E402 # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -57,9 +56,9 @@ # The short X.Y version. # realpath() with make your script run, even if you symlink it :) -version = tendo_version.__version__ +version = __version__ # The full version, including alpha/beta/rc tags. -release = tendo_version.__version__ +release = __version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/requirements-dev.txt b/requirements-dev.txt index 1a97b09..a46ced1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,12 +1,11 @@ +hacking>=1.0.0 + Sphinx autopep8 coveralls pep8 -flake8 -flake8-docstrings pytest>=2.6.0 pytest-cov -pytest-pep8 pytest-xdist pytest-instafail wheel diff --git a/setup.cfg b/setup.cfg index ae4a8d8..9f17b2e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,6 +38,7 @@ packages = [entry_points] pbr.config.drivers = plain = pbr.cfg.driver:Plain + [bdist_wheel] universal = 1 @@ -52,7 +53,7 @@ upload-dir = docs/build/html [tool:pytest] norecursedirs = . .svn _build tmp* lib/third lib *.egg bin distutils build docs demo python_files = *.py -addopts = -p no:xdist --ignore=setup.py --tb=long --capture=fd -rxX --maxfail=10 --pep8 tendo +addopts = -p no:xdist --ignore=setup.py --tb=long --capture=fd -rxX --maxfail=10 tendo # --maxfail=2 -n4 # -n4 runs up to 4 parallel procs # --maxfail=2 fail fast, dude @@ -61,34 +62,14 @@ addopts = -p no:xdist --ignore=setup.py --tb=long --capture=fd -rxX --maxfail=1 # these are important for distributed testing, to speedup their execution we minimize what we sync rsyncdirs = . tendo demo docs rsyncignore = .hg .git -pep8ignore = E501 E265 E127 E901 E128 E402 -filterwarnings = default - ignore:.*mode is deprecated:Warning - ignore:unclosed file.*:Warning - ignore:can't resolve package from.*:Warning +filterwarnings = + default + ignore:.*mode is deprecated:Warning + ignore:unclosed file.*:Warning + ignore:can't resolve package from.*:Warning [flake8] -exclude = migrations,__pycache__,build,bmll/config.py,env,src,.tox -# the only additional ignores are the docstrings ones Dxx, the other. -# ones are the default ones. -# see http://pep8.readthedocs.io/en/latest/intro.html -ignore = E121,E123,E126,E133,E226,E241,E242,E704,W503,D100,D101,D102,D103,D104,D105,D200,D202,D203,D204,D205,D207,D210,D211,D300,D301,D400,D401 -max-line-length=1024 - -[pep8] -exclude=build,lib,.tox,third,*.egg,docs,packages -;filename= -;select -ignore=E501,E265,E402 +enable-extensions = H106,H203,H204,H205,H210,H904 +exclude = __pycache__,build,src,.tox +ignore = D max-line-length=1024 -count=1 -;format -;quiet -;show-pep8 -;show-source -statistics=1 -;verbose=1 - -;PEP8_OPTS="--filename=*.py --exclude=lib --ignore=E501 scripts" -;pep8 $PEP8_OPTS --show-source --repeat -;pep8 --statistics -qq $PEP8_OPTS diff --git a/tendo/__init__.py b/tendo/__init__.py index 4752128..239689d 100644 --- a/tendo/__init__.py +++ b/tendo/__init__.py @@ -2,20 +2,22 @@ # Licensed to PSF under a Contributor Agreement. # See http://www.python.org/psf/license for licensing details. from __future__ import absolute_import -from .version import __version__ import sys +from pbr.version import VersionInfo + + +_v = VersionInfo('wstools').semantic_version() +__version__ = _v.release_string() +version_info = _v.version_tuple() + __author__ = "Sorin Sbarnea" -__copyright__ = "Copyright 2010-2015, Sorin Sbarnea" +__copyright__ = "Copyright 2010-2018, Sorin Sbarnea" __email__ = "sorin.sbarnea@gmail.com" __status__ = "Production" -__date__ = "2015-07-28" __all__ = ('tee', 'colorer', 'unicode', - 'execfile2', 'singleton', 'ansiterm', 'version', '__version__') + 'execfile2', 'singleton', 'ansiterm', '__version__') -""" -Tendo is tested with Python 2.5-3.4 -""" if sys.hexversion < 0x02050000: sys.exit("Python 2.5 or newer is required by tendo module.") diff --git a/tendo/ansiterm.py b/tendo/ansiterm.py index fe150a7..1425a5e 100755 --- a/tendo/ansiterm.py +++ b/tendo/ansiterm.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # Originally from http://waf.googlecode.com/svn/trunk/waflib/ansiterm.py -import sys import os +import sys try: if (not sys.stderr.isatty()) or (not sys.stdout.isatty()): raise ValueError('not a tty') @@ -37,7 +37,7 @@ class CONSOLE_CURSOR_INFO(Structure): try: _type = unicode - except: + except Exception: _type = str def to_int(number, default): diff --git a/tendo/colorer.py b/tendo/colorer.py index 18ebebe..a5fea4c 100755 --- a/tendo/colorer.py +++ b/tendo/colorer.py @@ -1,7 +1,5 @@ #!/usr/bin/env python # encoding: utf-8 -# Author: sorin sbarnea -# License: public domain """ Colorer does enable colored logging messages by using `ANSI escape sequences <http://en.wikipedia.org/wiki/ANSI_escape_code>`_. @@ -16,13 +14,13 @@ ... logging.info("gray line") ... logging.debug("magenta line") """ -import logging import copy +import logging import os +import six import sys import tempfile import unittest -import six if (hasattr(sys.stderr, "isatty") and sys.stderr.isatty()) or \ ('TERM' in os.environ.keys() and os.environ['TERM'] in ['linux']) or \ diff --git a/tendo/execfile2.py b/tendo/execfile2.py index e3d1bf5..8c09ae7 100755 --- a/tendo/execfile2.py +++ b/tendo/execfile2.py @@ -1,9 +1,9 @@ #!/usr/bin/env python import os +import shlex import sys -import unittest import tempfile -import shlex +import unittest if sys.hexversion > 0x03000000: def execfile(file, globals=globals(), locals=locals()): @@ -14,8 +14,8 @@ def execfile(file, globals=globals(), locals=locals()): def execfile2(filename, _globals=dict(), _locals=dict(), cmd=None, quiet=False): - """ - Execute a Python script using :py:func:`execfile`. + """Execute a Python script using :py:func:`execfile`. + In addition to Python :py:func:`execfile` this method can temporary change the argv params. This enables you to call an external python script that requires @@ -28,7 +28,6 @@ def execfile2(filename, _globals=dict(), _locals=dict(), cmd=None, quiet=False): - 0 - if succesfull; this applies if script receives SystemExit with error code 0 - 1 - if SystemExit does not contain an error code or if other Exception is received. - x - the SystemExit error code (if present) - """ _globals['__name__'] = '__main__' saved_argv = sys.argv # we save sys.argv @@ -74,32 +73,32 @@ def _exec_py_code(self, code, cmd=None): def test_normal_execution(self): exit_code = self._exec_py_code("") - self.assertTrue(exit_code == 0) + self.assertEqual(exit_code, 0) def test_bad_code(self): exit_code = self._exec_py_code("bleah") - self.assertTrue(exit_code == 1) + self.assertEqual(exit_code, 1) def test_sys_exit_0(self): exit_code = self._exec_py_code("import sys; sys.exit(0)") - self.assertTrue(exit_code == 0) + self.assertEqual(exit_code, 0) def test_sys_exit_5(self): exit_code = self._exec_py_code("import sys; sys.exit(5)") - self.assertTrue(exit_code == 5) + self.assertEqual(exit_code, 5) def test_sys_exit_text(self): exit_code = self._exec_py_code("import sys; sys.exit('bleah')") - self.assertTrue(exit_code == 1) + self.assertEqual(exit_code, 1) def test_raised_exception(self): exit_code = self._exec_py_code("raise Exception('bleah')") - self.assertTrue(exit_code == 1) + self.assertEqual(exit_code, 1) def test_command_line(self): exit_code = self._exec_py_code( "import sys\nif len(sys.argv)==2 and sys.argv[1]=='doh!': sys.exit(-1)", cmd="doh!") - self.assertTrue(exit_code == -1) + self.assertEqual(exit_code, -1) if __name__ == "__main__": unittest.main() diff --git a/tendo/singleton.py b/tendo/singleton.py index 70dd94f..a62351e 100755 --- a/tendo/singleton.py +++ b/tendo/singleton.py @@ -1,20 +1,21 @@ #! /usr/bin/env python -import sys +import logging +from multiprocessing import Process import os +import sys import tempfile import unittest -import logging -from multiprocessing import Process class SingleInstanceException(BaseException): pass -class SingleInstance: +class SingleInstance(object): + + """Class that can be instantiated only once per machine. - """ If you want to prevent your script from running in parallel just instantiate SingleInstance() class. If is there another instance already running it will throw a `SingleInstanceException`. >>> import tendo @@ -66,8 +67,8 @@ def __init__(self, flavor_id=""): self.initialized = True def __del__(self): - import sys import os + import sys if not self.initialized: return try: diff --git a/tendo/tee.py b/tendo/tee.py index d0ce571..10dcd15 100755 --- a/tendo/tee.py +++ b/tendo/tee.py @@ -1,17 +1,15 @@ #!/usr/bin/env python # encoding: utf-8 -# Author: sorin sbarnea -# License: public domain import codecs import logging import os import pipes -import sys +from six import string_types import subprocess -import types +import sys import time +import types import unittest -from six import string_types global logger global stdout @@ -30,8 +28,8 @@ def quote_command(cmd): - """ - This function does assure that the command line is entirely quoted. + """This function does assure that the command line is entirely quoted. + This is required in order to prevent getting "The input line is too long" error message. """ if not (os.name == "nt" or os.name == "dos"): @@ -45,7 +43,7 @@ def quote_command(cmd): def system2(cmd, cwd=None, logger=_sentinel, stdout=_sentinel, log_command=_sentinel, timing=_sentinel): # def tee(cmd, cwd=None, logger=tee_logger, console=tee_console): - """ Works exactly like :func:`system` but it returns both the exit code and the output as a list of lines. + """Works exactly like :func:`system` but it returns both the exit code and the output as a list of lines. This method returns a tuple: (return_code, output_lines_as_list). The return code of 0 means success. """ @@ -161,7 +159,7 @@ def secondsToStr(t): def system(cmd, cwd=None, logger=None, stdout=None, log_command=_sentinel, timing=_sentinel): - """ This works similar to :py:func:`os.system` but add some useful optional parameters. + """This works similar to :py:func:`os.system` but add some useful optional parameters. * ``cmd`` - command to be executed * ``cwd`` - optional working directory to be set before running cmd @@ -185,8 +183,8 @@ def system(cmd, cwd=None, logger=None, stdout=None, log_command=_sentinel, timin class testTee(unittest.TestCase): def test_1(self): - """ - No CMD os.system() + """No CMD os.system() + 1 sort /? ok ok 2 "sort" /? ok ok 3 sort "/?" ok ok diff --git a/tendo/unicode.py b/tendo/unicode.py index bd195c3..04eaec3 100755 --- a/tendo/unicode.py +++ b/tendo/unicode.py @@ -1,11 +1,11 @@ #!/usr/bin/python import codecs -import sys -import unittest +import inspect import logging -import tempfile import os -import inspect +import sys +import tempfile +import unittest import six """ @@ -20,8 +20,7 @@ def open(filename, mode='r', bufsize=-1, fallback_encoding='utf_8'): - """ - This replaces Python original function with an improved version that is Unicode aware. + """This replaces Python original function with an improved version that is Unicode aware. The new `open()` does change behaviour only for text files, not binary. @@ -39,7 +38,6 @@ def open(filename, mode='r', bufsize=-1, fallback_encoding='utf_8'): Files with BOM will be read properly as Unicode and the BOM will not be part of the text. If you do not specify the fallback_encoding, files without BOM will be read as `UTF-8` instead of `ascii`. - """ # Do not assign None to bufsize or mode because calling original open will # fail @@ -51,7 +49,7 @@ def open(filename, mode='r', bufsize=-1, fallback_encoding='utf_8'): f = open_old(filename, "rb") aBuf = bytes(f.read(4)) f.close() - except: + except Exception: aBuf = six.b('') if six.binary_type(aBuf[:3]) == six.b('\xEF\xBB\xBF'): f = codecs.open(filename, mode, "utf_8") @@ -117,9 +115,8 @@ def test_read_invalid_utf8(self): self.assertTrue(passed, "Unable to detect invalid utf8 file") def test_write_on_existing_utf8(self): - import shutil import filecmp - import os + import shutil (ftmp, fname_tmp) = tempfile.mkstemp() shutil.copyfile(os.path.join(self.dir, "tests/utf8.txt"), fname_tmp) f = open(fname_tmp, "a") # encoding not specified, should use utf-8 diff --git a/tendo/version.py b/tendo/version.py deleted file mode 100755 index bede63d..0000000 --- a/tendo/version.py +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env python -__version__ = "0.2.9" -__date__ = "2016-03-29" diff --git a/tox.ini b/tox.ini index 9556822..3e73d1a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,21 @@ [tox] minversion = 2.3.1 -envlist = {py27,py34,py35,py36}-{win,linux,darwin} +envlist = lint,docs,{py27,py34,py35,py36} skip_missing_interpreters = true -#addopts = --ignore=setup.py +tox_pyenv_fallback=True +ignore_errors=False + +[testenv] +sitepackages=False +passenv = + PY_* + PYENV_VERSION + RTOX* + SSH_AUTH_SOCK + +commands= + python -m pip -q install -rrequirements-dev.txt -rrequirements.txt + python -m pytest --cov-report xml --cov tendo --pyargs tendo [testenv:docs] basepython=python @@ -12,17 +25,9 @@ deps= six docutils commands= - sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html - -[testenv] -sitepackages=False -platform = - win: windows - linux: linux - darwin: darwin -deps= - -rrequirements.txt - -rrequirements-dev.txt + sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html +[testenv:lint] commands= - python -m pytest --cov-report xml --cov tendo --pyargs tendo + python -m pip -q install -rrequirements-dev.txt -rrequirements.txt + flake8