Merge branch 'services_init' into 17_positional_options

Conflicts:
	.coveragerc

Author: hartym

.codacy.yml

@@ -0,0 +1,4 @@
+---
+exclude_paths:
+  - bonobo/examples/
+  - bonobo/ext/

.coveragerc

@@ -2,6 +2,7 @@
 branch = True
 omit =
     bonobo/examples/**
+    bonobo/ext/**
 
 [report]
 # Regexes for lines to exclude from consideration

.gitignore

@@ -25,6 +25,7 @@
 /.idea
 /.release
 /bonobo.iml
+/bonobo/examples/work_in_progress/
 /bonobo/ext/jupyter/js/node_modules/
 /build/
 /coverage.xml

Makefile

@@ -1,7 +1,7 @@
 # This file has been auto-generated.
 # All changes will be lost, see Projectfile.
 #
-# Updated at 2017-04-30 10:12:39.793759
+# Updated at 2017-05-01 08:35:15.162008
 
 PYTHON ?= $(shell which python)
 PYTHON_BASENAME ?= $(shell basename $(PYTHON))
@@ -10,6 +10,7 @@ PYTHON_REQUIREMENTS_DEV_FILE ?= requirements-dev.txt
 QUICK ?= 
 VIRTUAL_ENV ?= .virtualenv-$(PYTHON_BASENAME)
 PIP ?= $(VIRTUAL_ENV)/bin/pip
+PIP_INSTALL_OPTIONS ?= 
 PYTEST ?= $(VIRTUAL_ENV)/bin/pytest
 PYTEST_OPTIONS ?= --capture=no --cov=bonobo --cov-report html
 SPHINX_OPTS ?= 
@@ -24,13 +25,13 @@ YAPF_OPTIONS ?= -rip
 # Installs the local project dependencies.
 install: $(VIRTUAL_ENV)
 	if [ -z "$(QUICK)" ]; then \
-	    $(PIP) install -U pip wheel -r $(PYTHON_REQUIREMENTS_FILE) ; \
+	    $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_FILE) ; \
 	fi
 
 # Installs the local project dependencies, including development-only libraries.
 install-dev: $(VIRTUAL_ENV)
 	if [ -z "$(QUICK)" ]; then \
-	    $(PIP) install -U pip wheel -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \
+	    $(PIP) install -U pip wheel $(PIP_INSTALL_OPTIONS) -r $(PYTHON_REQUIREMENTS_DEV_FILE) ; \
 	fi
 
 # Cleans up the local mess.

appveyor.yml

@@ -24,6 +24,8 @@ environment:
       PYTHON_VERSION: "3.6.1"
       PYTHON_ARCH: "64"
 
+build: false
+
 install:
   # If there is a newer build queued for the same PR, cancel this one.
   # The AppVeyor 'rollout builds' option is supposed to serve the same
@@ -61,18 +63,18 @@ install:
   # compiled extensions and are not provided as pre-built wheel packages,
   # pip will build them from source using the MSVC compiler matching the
   # target Python version and architecture
-  - "%CMD_IN_ENV% pip install -e ."
+  - "%CMD_IN_ENV% pip install -e .[dev]"
 
 test_script:
   # Run the project tests
   - "%CMD_IN_ENV% pytest --capture=no --cov=bonobo --cov-report html"
 
-after_test:
-  # If tests are successful, create binary packages for the project.
-  - "%CMD_IN_ENV% python setup.py bdist_wheel"
-  - "%CMD_IN_ENV% python setup.py bdist_wininst"
-  - "%CMD_IN_ENV% python setup.py bdist_msi"
-  - ps: "ls dist"
+#after_test:
+# If tests are successful, create binary packages for the project.
+#  - "%CMD_IN_ENV% python setup.py bdist_wheel"
+#  - "%CMD_IN_ENV% python setup.py bdist_wininst"
+#  - "%CMD_IN_ENV% python setup.py bdist_msi"
+#  - ps: "ls dist"
 
 artifacts:
   # Archive the generated packages in the ci.appveyor.com build report.

bonobo/__init__.py

@@ -11,6 +11,9 @@
 assert (sys.version_info >= (3, 5)), 'Python 3.5+ is required to use Bonobo.'
 from bonobo._api import *
 from bonobo._api import __all__
+from bonobo._version import __version__
+
+__all__ = ['__version__'] + __all__
+__version__ = __version__
 
-__all__ = __all__
 del sys

bonobo/_api.py

@@ -1,49 +1,105 @@
-from bonobo._version import __version__
-
-__all__ = [
-    '__version__',
-]
+import warnings
 
+from bonobo.basics import Limit, PrettyPrint, Tee, count, identity, noop, pprint
+from bonobo.strategies import create_strategy
 from bonobo.structs import Bag, Graph
+from bonobo.util.objects import get_name
+from bonobo.io import CsvReader, CsvWriter, FileReader, FileWriter, JsonReader, JsonWriter
 
-__all__ += ['Bag', 'Graph']
+__all__ = []
 
-# Filesystem. This is a shortcut from the excellent filesystem2 library, that we make available there for convenience.
-from fs import open_fs as _open_fs
 
-open_fs = lambda url, *args, **kwargs: _open_fs(str(url), *args, **kwargs)
-__all__ += ['open_fs']
+def register_api(x, __all__=__all__):
+    __all__.append(get_name(x))
+    return x
 
-# Basic transformations.
-from bonobo.basics import *
-from bonobo.basics import __all__ as _all_basics
 
-__all__ += _all_basics
+def register_api_group(*args):
+    for attr in args:
+        register_api(attr)
 
-# Execution strategies.
-from bonobo.strategies import create_strategy
 
-__all__ += ['create_strategy']
+@register_api
+def run(graph, *chain, strategy=None, plugins=None, services=None):
+    """
+    Main entry point of bonobo. It takes a graph and creates all the necessary plumbery around to execute it.
+    
+    The only necessary argument is a :class:`Graph` instance, containing the logic you actually want to execute.
+    
+    By default, this graph will be executed using the "threadpool" strategy: each graph node will be wrapped in a
+    thread, and executed in a loop until there is no more input to this node.
+    
+    You can provide plugins factory objects in the plugins list, this function will add the necessary plugins for
+    interactive console execution and jupyter notebook execution if it detects correctly that it runs in this context.
+    
+    You'll probably want to provide a services dictionary mapping service names to service instances.
+    
+    :param Graph graph: The :class:`Graph` to execute.
+    :param str strategy: The :class:`bonobo.strategies.base.Strategy` to use.
+    :param list plugins: The list of plugins to enhance execution.
+    :param dict services: The implementations of services this graph will use.
+    :return bonobo.execution.graph.GraphExecutionContext:
+    """
+    if len(chain):
+        warnings.warn('DEPRECATED. You should pass a Graph instance instead of a chain.')
+        from bonobo import Graph
+        graph = Graph(graph, *chain)
 
-# Extract and loads from stdlib.
-from bonobo.io import *
-from bonobo.io import __all__ as _all_io
+    strategy = create_strategy(strategy)
+    plugins = plugins or []
 
-__all__ += _all_io
+    if _is_interactive_console():
+        from bonobo.ext.console import ConsoleOutputPlugin
+        if ConsoleOutputPlugin not in plugins:
+            plugins.append(ConsoleOutputPlugin)
 
+    if _is_jupyter_notebook():
+        from bonobo.ext.jupyter import JupyterOutputPlugin
+        if JupyterOutputPlugin not in plugins:
+            plugins.append(JupyterOutputPlugin)
 
-# XXX This may be belonging to the bonobo.examples package.
-def get_examples_path(*pathsegments):
-    import os
-    import pathlib
-    return str(pathlib.Path(os.path.dirname(__file__), 'examples', *pathsegments))
+    return strategy.execute(graph, plugins=plugins, services=services)
 
 
-def open_examples_fs(*pathsegments):
-    return open_fs(get_examples_path(*pathsegments))
+# bonobo.structs
+register_api_group(Bag, Graph)
+
+# bonobo.strategies
+register_api(create_strategy)
+
+
+# Shortcut to filesystem2's open_fs, that we make available there for convenience.
+@register_api
+def open_fs(fs_url, *args, **kwargs):
+    """
+    Wraps :func:`fs.open_fs` function with a few candies.
+    
+    :param str fs_url: A filesystem URL
+    :param parse_result: A parsed filesystem URL.
+    :type parse_result: :class:`ParseResult`
+    :param bool writeable: True if the filesystem must be writeable.
+    :param bool create: True if the filesystem should be created if it does not exist.
+    :param str cwd: The current working directory (generally only relevant for OS filesystems).
+    :param str default_protocol: The protocol to use if one is not supplied in the FS URL (defaults to ``"osfs"``).
+    :returns: :class:`~fs.base.FS` object
+    """
+    from fs import open_fs as _open_fs
+    return _open_fs(str(fs_url), *args, **kwargs)
 
-__all__.append(get_examples_path.__name__)
-__all__.append(open_examples_fs.__name__)
+
+# bonobo.basics
+register_api_group(
+    Limit,
+    PrettyPrint,
+    Tee,
+    count,
+    identity,
+    noop,
+    pprint,
+)
+
+# bonobo.io
+register_api_group(CsvReader, CsvWriter, FileReader, FileWriter, JsonReader, JsonWriter)
 
 
 def _is_interactive_console():
@@ -58,27 +114,13 @@ def _is_jupyter_notebook():
         return False
 
 
-# @api
-def run(graph, *chain, strategy=None, plugins=None, services=None):
-    if len(chain):
-        warnings.warn('DEPRECATED. You should pass a Graph instance instead of a chain.')
-        from bonobo import Graph
-        graph = Graph(graph, *chain)
-
-    strategy = create_strategy(strategy)
-    plugins = []
-
-    if _is_interactive_console():
-        from bonobo.ext.console import ConsoleOutputPlugin
-        if ConsoleOutputPlugin not in plugins:
-            plugins.append(ConsoleOutputPlugin)
-
-    if _is_jupyter_notebook():
-        from bonobo.ext.jupyter import JupyterOutputPlugin
-        if JupyterOutputPlugin not in plugins:
-            plugins.append(JupyterOutputPlugin)
-
-    return strategy.execute(graph, plugins=plugins, services=services)
+@register_api
+def get_examples_path(*pathsegments):
+    import os
+    import pathlib
+    return str(pathlib.Path(os.path.dirname(__file__), 'examples', *pathsegments))
 
 
-__all__.append(run.__name__)
+@register_api
+def open_examples_fs(*pathsegments):
+    return open_fs(get_examples_path(*pathsegments))

bonobo/basics.py

@@ -4,7 +4,6 @@
 from colorama import Fore, Style
 
 from bonobo.config.processors import contextual
-from bonobo.constants import NOT_MODIFIED
 from bonobo.structs.bags import Bag
 from bonobo.util.objects import ValueHolder
 from bonobo.util.term import CLEAR_EOL
@@ -25,6 +24,7 @@ def identity(x):
 
 
 def Limit(n=10):
+    from bonobo.constants import NOT_MODIFIED
     i = 0
 
     def _limit(*args, **kwargs):
@@ -38,6 +38,8 @@ def _limit(*args, **kwargs):
 
 
 def Tee(f):
+    from bonobo.constants import NOT_MODIFIED
+
     @functools.wraps(f)
     def wrapped(*args, **kwargs):
         nonlocal f
@@ -63,6 +65,8 @@ def _count_counter(self, context):
 
 
 def PrettyPrint(title_keys=('title', 'name', 'id'), print_values=True, sort=True):
+    from bonobo.constants import NOT_MODIFIED
+
     def _pprint(*args, **kwargs):
         nonlocal title_keys, sort, print_values
 
@@ -98,4 +102,5 @@ def _pprint(*args, **kwargs):
 
 
 def noop(*args, **kwargs):  # pylint: disable=unused-argument
+    from bonobo.constants import NOT_MODIFIED
     return NOT_MODIFIED

bonobo/commands/init.py

@@ -3,14 +3,12 @@
 
 def execute():
     try:
-        import edgy.project
+        from edgy.project.__main__ import handle_init
     except ImportError as exc:
         raise ImportError(
             'You must install "edgy.project" to use this command.\n\n $ pip install edgy.project\n'
         ) from exc
 
-    from edgy.project.__main__ import handle_init
-
     return handle_init(os.path.join(os.getcwd(), 'Projectfile'))
 
 

bonobo/commands/run.py

@@ -20,7 +20,7 @@ def get_default_services(filename, services=None):
         }
         try:
             exec(code, context)
-        except Exception as exc:
+        except Exception:
             raise
         return {
             **context[DEFAULT_SERVICES_ATTR](),
@@ -55,7 +55,7 @@ def execute(file, quiet=False):
         'but it is something that will be implemented in the future.\n\nExpected: 1, got: {}.'
     ).format(len(graphs))
 
-    name, graph = list(graphs.items())[0]
+    graph = list(graphs.values())[0]
 
     # todo if console and not quiet, then add the console plugin
     # todo when better console plugin, add it if console and just disable display

bonobo/config/configurables.py

@@ -83,4 +83,4 @@ def __init__(self, *args, **kwargs):
             )
 
         for name, value in kwargs.items():
-            setattr(self, name, kwargs[name])
+            setattr(self, name, value)

bonobo/config/processors.py

@@ -52,9 +52,9 @@ def contextual(cls_or_func):
         setattr(cls_or_func, _CONTEXT_PROCESSORS_ATTR, [])
 
     _processors = getattr(cls_or_func, _CONTEXT_PROCESSORS_ATTR)
-    for name, value in cls_or_func.__dict__.items():
-        if isinstance(value, ContextProcessor):
-            _processors.append(value)
+    for processor in cls_or_func.__dict__.values():
+        if isinstance(processor, ContextProcessor):
+            _processors.append(processor)
 
     # This is needed for python 3.5, python 3.6 should be fine, but it's considered an implementation detail.
     _processors.sort(key=lambda proc: proc._creation_counter)

bonobo/examples/.style.yapf

@@ -0,0 +1,4 @@
+[style]
+based_on_style = pep8
+column_limit = 74
+dedent_closing_brackets = true

bonobo/examples/datasets/coffeeshops.py

@@ -1,3 +1,18 @@
+"""
+Extracts a list of parisian bars where you can buy a coffee for a reasonable price, and store them in a flat text file.
+
+.. graphviz::
+
+    digraph {
+        rankdir = LR;
+        stylesheet = "../_static/graphs.css";
+
+        BEGIN [shape="point"];
+        BEGIN -> "ODS()" -> "transform" -> "FileWriter()";
+    }
+
+"""
+
 import bonobo
 from bonobo.commands.run import get_default_services
 from bonobo.ext.opendatasoft import OpenDataSoftAPI