Merge pull request #1 from serge-sans-paille/master

Merge upstream into master

Author: ashwinvis

.appveyor.yml

@@ -1,5 +1,5 @@
 image:
-- Previous Visual Studio 2017
+- Visual Studio 2017
 
 platform:
 - x64

.gitignore

@@ -6,6 +6,7 @@ website/*.html
 website/license.txt
 pythran/nt2
 pythran/boost
+pythran/xsimd
 parsetab.py
 
 .eggs/*

.travis.yml

@@ -12,7 +12,6 @@ addons:
             - g++-5
             - clang-3.5
             - clang-format-3.6
-            - libgmp-dev
             - libatlas-base-dev
 
 install:
@@ -22,31 +21,31 @@ install:
 
 env:
     - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS=-g TESTCASE='pythran/tests/test_[ab]*.py pythran/tests/test_c[b-z]*.py pythran/tests/test_numpy_[a-eg-r]*.py pythran/tests/test_numpy_f[a-t]*.py'
-    - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS="-DUSE_BOOST_SIMD -march=native" TESTCASE='pythran/tests/test_[ab]*.py pythran/tests/test_c[b-z]*.py pythran/tests/test_numpy_[a-eg-r]*.py pythran/tests/test_numpy_f[a-t]*.py'
+    - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS="-DUSE_XSIMD -march=native" TESTCASE='pythran/tests/test_[ab]*.py pythran/tests/test_c[b-z]*.py pythran/tests/test_numpy_[a-eg-r]*.py pythran/tests/test_numpy_f[a-t]*.py'
     - CC=gcc-5 CXX=g++-5 CXXFLAGS="-fopenmp" TESTCASE='pythran/tests/test_[ab]*.py pythran/tests/test_c[b-z]*.py pythran/tests/test_numpy_[a-eg-r]*.py pythran/tests/test_numpy_f[a-t]*.py'
 
     - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS=-g TESTCASE='pythran/tests/test_ca*.py'
-    - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS="-DUSE_BOOST_SIMD -march=native" TESTCASE='pythran/tests/test_ca*.py'
+    - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS="-DUSE_XSIMD -march=native" TESTCASE='pythran/tests/test_ca*.py'
     - CC=gcc-5 CXX=g++-5 CXXFLAGS="-fopenmp" TESTCASE='pythran/tests/test_ca*.py'
-    - CC=gcc-5 CXX=g++-5 CXXFLAGS="-DUSE_BOOST_SIMD -march=native -fopenmp" TESTCASE='pythran/tests/test_ca*.py'
+    - CC=gcc-5 CXX=g++-5 CXXFLAGS="-DUSE_XSIMD -march=native -fopenmp" TESTCASE='pythran/tests/test_ca*.py'
 
     - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS=-g TESTCASE='pythran/tests/test_[d-m]*.py pythran/tests/test_n[a-t]*.py'
-    - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS="-DUSE_BOOST_SIMD -march=native" TESTCASE='pythran/tests/test_[d-m]*.py pythran/tests/test_n[a-t]*.py'
+    - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS="-DUSE_XSIMD -march=native" TESTCASE='pythran/tests/test_[d-m]*.py pythran/tests/test_n[a-t]*.py'
     - CC=gcc-5 CXX=g++-5 CXXFLAGS="-fopenmp" TESTCASE='pythran/tests/test_[d-m]*.py pythran/tests/test_n[a-t]*.py'
 
     - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS=-g TESTCASE='pythran/tests/test_numpy_f[u-z]*.py'
-    - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS="-DUSE_BOOST_SIMD -march=native" TESTCASE='pythran/tests/test_numpy_f[u-z]*.py'
+    - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS="-DUSE_XSIMD -march=native" TESTCASE='pythran/tests/test_numpy_f[u-z]*.py'
     - CC=gcc-5 CXX=g++-5 CXXFLAGS="-fopenmp" TESTCASE='pythran/tests/test_numpy_f[u-z]*.py'
 
     - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS=-g TESTCASE='pythran/tests/test_numpy_[s-z]*.py'
-    - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS="-DUSE_BOOST_SIMD -march=native" TESTCASE='pythran/tests/test_numpy_[s-z]*.py'
+    - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS="-DUSE_XSIMD -march=native" TESTCASE='pythran/tests/test_numpy_[s-z]*.py'
     - CC=gcc-5 CXX=g++-5 CXXFLAGS="-fopenmp" TESTCASE='pythran/tests/test_numpy_[s-z]*.py'
 
-    - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS=-g TESTCASE='pythran/tests/test_[o-z]*.py'
-    - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS="-DUSE_BOOST_SIMD -march=native" TESTCASE='pythran/tests/test_[o-z]*.py'
-    - CC=gcc-5 CXX=g++-5 CXXFLAGS="-fopenmp" TESTCASE='pythran/tests/test_[o-z]*.py'
+    - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS=-g TESTCASE='pythran/tests/test_[o-w]*.py'
+    - CC=clang-3.5 CXX=clang++-3.5 CXXFLAGS="-DUSE_XSIMD -march=native" TESTCASE='pythran/tests/test_[o-w]*.py'
+    - CC=gcc-5 CXX=g++-5 CXXFLAGS="-fopenmp" TESTCASE='pythran/tests/test_[o-w]*.py'
 
-    - PYTHON_VERSION=2 NOTEBOOK=true
+    - PYTHRAN_DOC=true
 
       #    - TESTCASE=pythran/tests/test_base.py
       #    - TESTCASE=pythran/tests/test_bisect.py

Changelog

@@ -1,3 +1,59 @@
+2018-11-06 Serge Guelton <[email protected]>
+
+	* Moving to xsimd as vectorization engine, requires -DUSE_XSIMD
+
+	* Better support of view conversion from Python to Pythran
+
+	* Improved Cython integration
+
+	* Improved documentation, add exemple section
+
+	* Updated ArchLinux Packaging
+
+	* Remove useless warnings during compilation
+
+	* Faster ``abs(x**2)`` for complex numbers
+
+	* IPython magic now accepts most optimization flags
+
+	* Automatic detection of partially (or fully) constant shape for arrays
+
+	* Add ignoreflags settings to .pythranrc to skip some compiler flags
+
+	* Quad number (aka long double) basic support
+
+	* And many minor bugfixes, as usual :-)
+
+
+2018-09-16 Serge Guelton <[email protected]>
+
+	* Decent support for ``is None`` pattern
+
+	* Bessel functions, aka scipy.special.stuff
+
+	* ``__builtin__.slice`` support
+
+	* ``np.unravel_index``, ``np.(u)intc`` support
+
+	* Cleanup and fix issues around sdist / build_ext and stuff
+
+	* O(1) View conversion to Python
+
+	* Dropped big int support
+
+	* Speed improvement for ``np.argmin``, ``np.argmax`` and square of complex
+
+	* Allow partially (or fully) constant shape for arrays
+
+	* Deterministic header inclusion (toward reproducible build?)
+
+	* Better error report on argument mismatch
+
+	* Better support for view arguments (but still lotta missing :-/)
+
+	* Better Windows support (fix compiler-specific bugs) incuding OpenMP
+
+
 2018-06-06 Serge Guelton <[email protected]>
 
 	* Much better support of fancy indexing, still not 100% support though
@@ -13,7 +69,7 @@
 	* Most tests now work on Python 2 and Python 3
 
 	* Many bugfixes triggered by the bug report of Yann Diorcet, Jean Laroche
-	and David Menéndez Hurtado, Fabien Rozar, C. Claus
+	  and David Menéndez Hurtado, Fabien Rozar, C. Claus
 
 
 2018-04-23 Serge Guelton <[email protected]>

README.rst

@@ -35,7 +35,7 @@ Using ``pip``
 
    Pythran depends on a few Python modules and several C++ libraries. On a debian-like platform, run::
 
-        $> sudo apt-get install libgmp-dev libatlas-base-dev
+        $> sudo apt-get install libatlas-base-dev
         $> sudo apt-get install python-dev python-ply python-networkx python-numpy
 
 2. Use ``easy_install`` or ``pip``::
@@ -57,9 +57,6 @@ Mac OSX
 
 Using brew (http://brew.sh/)::
 
-    $> brew install gmp
-    $> brew install cmake
-
     $> easy_install pip
 
     $> pip install numpy pythran
@@ -73,9 +70,9 @@ Depending on your setup, you may need to add the following to your \\~/.pythranr
 ArchLinux
 =========
 
-Using `yaourt`::
+Using any working `AUR helper <https://wiki.archlinux.org/index.php/AUR_helpers>`_, say aurman::
 
-    $> yaourt -S python2-pythran-git
+    $> aurman -S python-pythran
 
 Windows
 =======

docs/CLI.rst

@@ -15,7 +15,7 @@ Firstly lets clear the working space::
 One of the most classic use case in Pythran is to generate a native .so module::
 
   $> printf '#pythran export foo()\n#pythran export msg\nmsg = \"hello world\"\ndef foo(): print(msg)' > cli_foo.py
-  $> pythran cli_foo.py
+  $> pythran cli_foo.py -o cli_foo.so
   $> ls cli_foo.so
   cli_foo.so
 
@@ -29,7 +29,7 @@ The generated native ``.so`` module can then be used with the Python interpreter
 Pythran version can be dumped through ``--version``::
 
   $> pythran --version 2>&1
-  0.8.6
+  0.9.0
 
 The module-level ``__pythran__`` variable indicates that the module loaded has been pythranized::
 
@@ -39,7 +39,7 @@ The module-level ``__pythran__`` variable indicates that the module loaded has b
 You can choose your optimization level by using ``-O`` flag::
 
   $> rm cli_foo.so
-  $> pythran cli_foo.py -O2
+  $> pythran cli_foo.py -O2 -o cli_foo.so
   $> ls cli_foo.so
   cli_foo.so
 
@@ -49,13 +49,13 @@ Out of curiosity, you can check the generated output::
 
 That's some heavily templated code ;-) Pythran can then compile it for you to a Python module::
 
-  $> pythran cli_foo.cpp
+  $> pythran cli_foo.cpp -o cli_foo.so
 
 Pythran can also generate raw C++ code, using the ``-e`` switch::
 
   $> pythran -e cli_foo.py -o cli_foo.hpp
   $> printf '#include \"cli_foo.hpp\"\nusing namespace __pythran_cli_foo ; int main() { foo()(); return 0 ; }' > cli_foo.cpp
-  $> `pythran-config --compiler --cflags` -std=c++11 cli_foo.cpp -o cli_foo -UUSE_GMP
+  $> `pythran-config --compiler --cflags` -std=c++11 cli_foo.cpp -o cli_foo
   $> ./cli_foo
   hello world
 

docs/EXAMPLES.rst

@@ -0,0 +1,7 @@
+Examples
+========
+
+.. toctree::
+
+    examples/Distutils Sample Project
+    examples/Third Party Libraries

docs/INTERNAL.rst

@@ -160,9 +160,9 @@ conditional::
     >>> _ = pm.apply(openmp.GatherOMPData, foo_tree)
     >>> print(pm.dump(backend.Python, foo_tree))
     def foo(n):
+        'omp parallel'
+        'omp single'
         if 1:
-            'omp parallel'
-            'omp single'
             s = 1
         return s
 

docs/MANUAL.rst

@@ -39,7 +39,6 @@ Prerequisite
 
 Pythran depends on the following packages:
 
-- GMP: https://gmplib.org/
 - ply: http://www.dabeaz.com/ply/
 - networkx: https://networkx.github.io/
 - numpy: http://www.numpy.org/
@@ -173,10 +172,11 @@ To the input file. You can compile it as the previous code::
 
 and you'll get a decent binary. But what you really want to do is::
 
-    $> pythran -fopenmp -march=avx arc_distance.py
-
-which basically tells the compiler to parallelize and vectorize loops. Then you'll get **really** fast code!
+    $> pythran -DUSE_XSIMD -fopenmp -march=native arc_distance.py
 
+which basically tells the compiler to parallelize and vectorize loops using
+whatever hardware available on your machine. Then you'll get **really** fast
+code!
 
 
 Concerning Pythran specifications
@@ -203,14 +203,15 @@ set), introduced by the ``list`` (resp. ``set``) keyword::
                   | argument_type set    # this is a set
                   | argument_type []+    # this is a ndarray, C-style
                   | argument_type [::]+    # this is a strided ndarray
-                  | argument_type [:,...,:]+ # this is a ndarray, Cython
+                  | argument_type [:,...,:]+ # this is a ndarray, Cython style
+                  | argument_type [:,...,3]+ # this is a ndarray, some dimension fixed
                   | argument_type:argument_type dict    # this is a dictionary
 
-    basic_type = bool | int | long | float | str | None
+    basic_type = bool | int | float | str | None
                | uint8 | uint16 | uint32 | uint64 | uintp
                | int8 | int16 | int32 | int64 | intp
-               | float32 | float64
-               | complex64 | complex128
+               | float32 | float64 | float128
+               | complex64 | complex128 | complex256
 
 The same syntax can be used to export global variable (in read only mode)::
 
@@ -315,9 +316,13 @@ When distributing a Python application with Pythran modules, you can either:
     import setuptools
     setuptools.dist.Distribution(dict(setup_requires='pythran'))
 
-    from pythran.dist import PythranExtension
+    from pythran.dist import PythranExtension, PythranBuildExt
     setup(...,
-          ext_modules=[PythranExtension("mymodule", ["mymodule.py"])])
+          ext_modules=[PythranExtension("mymodule", ["mymodule.py"])],
+          cmdclass={"build_ext": PythranBuildExt})
+
+``PythranBuildExt`` is optional, but necessary to build extensions with
+different C++ compilers.
 
 Capsule Corp
 ------------
@@ -330,7 +335,9 @@ line::
     #pythran export capsule foo(double*, doule)
 
 Note that pointer types are only supported within the context of a capsule, as
-they don't match any real Python type.
+they don't match any real Python type. **Any** Pythran type is valid as capsule
+parameter, but beware that non scalar or pointer types only make sense withing
+the Pythran context.
 
 Advanced Usage
 --------------
@@ -347,7 +354,8 @@ Tired of typing the same compiler switches again and again? Store them in
 ``$XDG_CONFIG_HOME/.pythranrc``!
 
 Wants to try your own compiler? Update the `CC` and `CXX` fields from your
-`pythranrc`, or set the same env variables to the right compilers.
+`pythranrc`, or set the same env variables to the right compilers. Environment
+variables have greater precedence than configuration file.
 
 The careful reader might have noticed the ``-p`` flag from the command line. It
 makes it possible to define your own optimization sequence::
@@ -439,23 +447,22 @@ This section contains compiler flags configuration. For education purpose, the d
 
 :``defines``:
 
-    Preprocessor definitions. Pythran is sensible to ``USE_BOOST_SIMD`` and
-    ``PYTHRAN_OPENMP_MIN_ITERATION_COUNT``. The former turns on Boost.simd
+    Preprocessor definitions. Pythran is sensible to ``USE_XSIMD`` and
+    ``PYTHRAN_OPENMP_MIN_ITERATION_COUNT``. The former turns on `xsimd <https://github.com/QuantStack/xsimd>`_
     vectorization and the latter controls the mimimal loop trip count to turn a
-    sequential loop into a parallel loop. The default is to set ``USE_GMP``, so
-    that Python's longs are represented using GMP.
+    sequential loop into a parallel loop.
 
 :``undefs``:
 
     Some preprocessor definitions to remove.
 
 :``include_dirs``:
 
-    Additionnal include directories to search for headers.
+    Additional include directories to search for headers.
 
 :``cflags``:
 
-    Additionnal random compiler flags (``-f``, ``-O``). Optimization flags generally
+    Additional random compiler flags (``-f``, ``-O``). Optimization flags generally
     go there. The default is to set ``-std=c++11`` for C++11 support.
 
 :``libs``:
@@ -470,12 +477,19 @@ This section contains compiler flags configuration. For education purpose, the d
 
 :``ldflags``:
 
-    Additionnal random linker flags.
+    Additional random linker flags.
 
 :``blas``:
 
     BLAS library to use. Default is ``blas``, but ``atlas`` or ``mkl`` are also viable choices.
 
+:``ignoreflags``:
+
+    Space-separated list of compiler flags that should not be forwarded to the
+    pythran backend compiler when inherited, for instance, from
+    ``python-config``. For instance ``-Wstrict-prototypes`` is a C-only option
+    that should be pruned.
+
 
 ``[pythran]``
 *************

docs/TUTORIAL.rst

@@ -76,7 +76,7 @@ subset of Python AST) into a C++ AST::
   >>> from pythran import backend
   >>> cxx = pm.dump(backend.Cxx, tree)
   >>> str(cxx)
-  '#include <pythonic/include/__builtin__/pythran/ifexp.hpp>\n#include <pythonic/__builtin__/pythran/ifexp.hpp>\nnamespace __pythran_tutorial_module\n{\n  struct fib\n  {\n    typedef void callable;\n    typedef void pure;\n    template <typename argument_type0 >\n    struct type\n    {\n      typedef typename pythonic::returnable<typename std::remove_cv<typename std::remove_reference<argument_type0>::type>::type>::type result_type;\n    }  \n    ;\n    template <typename argument_type0 >\n    typename type<argument_type0>::result_type operator()(argument_type0&& n) const\n    ;\n  }  ;\n  template <typename argument_type0 >\n  typename fib::type<argument_type0>::result_type fib::operator()(argument_type0&& n) const\n  {\n    return pythonic::__builtin__::pythran::ifexp((bool)(n < 2L), [&] () { return n; }, [&] () { return (fib()((n - 1L)) + fib()((n - 2L))); });\n  }\n}'
+  '#include <pythonic/include/__builtin__/pythran/ifexp.hpp>\n#include <pythonic/include/operator_/add.hpp>\n#include <pythonic/include/operator_/lt.hpp>\n#include <pythonic/__builtin__/pythran/ifexp.hpp>\n#include <pythonic/operator_/add.hpp>\n#include <pythonic/operator_/lt.hpp>\nnamespace __pythran_tutorial_module\n{\n  struct fib\n  {\n    typedef void callable;\n    typedef void pure;\n    template <typename argument_type0 >\n    struct type\n    {\n      typedef typename pythonic::returnable<typename std::remove_cv<typename std::remove_reference<argument_type0>::type>::type>::type result_type;\n    }  \n    ;\n    template <typename argument_type0 >\n    typename type<argument_type0>::result_type operator()(argument_type0&& n) const\n    ;\n  }  ;\n  template <typename argument_type0 >\n  typename fib::type<argument_type0>::result_type fib::operator()(argument_type0&& n) const\n  {\n    return pythonic::__builtin__::pythran::ifexp((bool)(pythonic::operator_::lt(n, 2L)), [&] () { return n; }, [&] () { return (pythonic::operator_::add(fib()((n - 1L)), fib()((n - 2L)))); });\n  }\n}'
 
 The above string is understandable by a C++11 compiler, but it quickly reaches the limit of our developer brain, so most of the time, we are more comfortable with the Python backend::
 
@@ -111,12 +111,12 @@ computation, if any::
   >>> tree = ast.parse("""
   ... def foo(x):
   ...     #omp critical
-  ...     a,b = bar(x)
+  ...     a,b = 1, x + 1
   ...     return a + b""")
   >>> _ = pm.apply(transformations.NormalizeTuples, tree)  # in-place
   >>> print(pm.dump(backend.Python, tree))
   def foo(x):
-      __tuple0 = bar(x)
+      __tuple0 = (1, (x + 1))
       a = __tuple0[0]
       b = __tuple0[1]
       return (a + b)
@@ -154,7 +154,7 @@ More complex ones rely on introspection to implement constant folding::
 
 One can also detect some common generator expression patterns to call the itertool module::
 
-  >>> norm = 'def norm(l): return sum(n*n for n in l)'
+  >>> norm = 'def norm(l): return __builtin__.sum(n*n for n in l)'
   >>> tree = ast.parse(norm)
   >>> _ = pm.apply(optim.ComprehensionPatterns, tree)
   >>> 'map' in pm.dump(backend.Python, tree)