Changes to support "pip install nexinfosys"

* Reduced number of packages. Specially, avoid use of Flask ("app" variable should not be used outside of "restful_service" package)
* "frontend" not included
* "setup.py" updated

.gitignore

@@ -6,6 +6,7 @@
 build/
 dist/
 nexinfosys_backend.egg-info/
+nexinfosys.egg-info/
 dump\.rdb
 nexinfosys/restful_service/nis_local.conf
 nexinfosys/restful_service/nis_docker_protagoras.conf

MANIFEST.in

@@ -1,4 +1,4 @@
-recursive-include nexinfosys/frontend *
+# recursive-include nexinfosys/frontend */
 exclude MANIFEST.in
 include requirements.txt
 include nexinfosys/restful_service/nis_local_dist.conf

README.md

@@ -31,6 +31,7 @@ It is an open system as it can be used to integrate MuSIASEM as a formalism in a
 - [Getting started](#getting-started)
   - [Features](#features)
   - [Installing and executing **nis-backend**](#installing-and-executing-nis-backend)
+    - [Pip package](#pip-package)
     - [Docker image](#docker-image)
     - [Source code](#source-code)
   - [Models with Commands](#models-with-commands)
@@ -62,7 +63,7 @@ It is an open system as it can be used to integrate MuSIASEM as a formalism in a
   - Sources, so several observations can be made about the same fact.
   - Times, in years or in months.
 - **Outputs**. Exportable as datasets (statistical cubes), graphs (networks), matrices, models (transformations of the input model), geolayers, scripts (Python or R).
-- **Deployment**. It can be deployed in multiple ways: personal computer (installer or pip), or server (Docker or pip).
+- **Deployment**. It can be deployed in multiple ways: personal computer (pip, source, Docker), or server (source or Docker).
 - **Configurable**. Deployment for server can be configured to use different database servers and REDIS for session management.
 - **Open**. All functionality can be accessed through a RESTful service. Behind the scenes, because HTTP protocol is stateless, a complex serialization/deserialization function combined with a key/value store and a browser session enable saving the state in memory after a service call ends, just to recover it when another invocation is done. 
 - **Two expertise levels**. Two components wrap the RESTful interface, **nis-frontend** and **nis-client**. These components match two expertise levels: **nis-frontend** does not require programming knowledge, while **nis-client** can be used in Python/R scripts.
@@ -71,20 +72,16 @@ It is an open system as it can be used to integrate MuSIASEM as a formalism in a
 
 **nis-backend** is a Python package. There are several options to have it installed.
 
-<!-- #### pip -->
+#### pip package
 
-<!-- Prepare package to upload it to pypi -->
+The pip version is obviously for execution as package, and does not include the frontend, the other two deployment options below include the frontend.
 
-<!--
 * Set a Python environment
 * Install the package with
 
-TO-DO `pip install nexinfosys`
-
-* Start the server
+`pip install nexinfosys`
 
-`python3 nexinfosys.restful_service.service_main.py`
--->
+* Use class class *NIS*, with methods similar to *NISClient* class in [nexinfosys-client](https://github.com/MAGIC-nexus/nis-python-client) package.
 
 #### Docker image
 

nexinfosys/__init__.py

@@ -132,6 +132,7 @@ def default_directories(path, tmp_path):
 # Flask Session (server side session)
 REDIS_HOST="filesystem:local_session"
 TESTING="True"
+ENABLE_CYTHON_OPTIMIZATIONS="True"
 SELF_SCHEMA=""
 FS_TYPE="WebDAV"
 FS_SERVER=""
@@ -204,6 +205,19 @@ def get_global_configuration_variable(key: str, default: str = None) -> str:
     return global_configuration.get(key.lower(), default)
 
 
+def set_global_configuration_variable(key: str, value: str):
+    """
+    Use carefully. The initial intention is to set the "ENABLE_CYTHON_OPTIMIZATIONS" variable to False
+
+    :param key:
+    :param value:
+    :return:
+    """
+    global global_configuration
+    if global_configuration:
+        global_configuration[key.lower()] = value
+
+
 def remove_quotes(s: str) -> str:
     return s.lstrip('\'"').rstrip('\'"')
 

nexinfosys/embedded_nis.py

@@ -406,6 +406,8 @@ def get_results(self, datasets: List[Tuple[str, str, str]]):
                     pos = ds_name.find(".")
                     extension = ds_name[pos + 1:]
                     ds_name = ds_name[:pos]
+                else:
+                    extension = None
 
                 if struc_type == "dataset":
                     ds, ctype, ok = get_dataset_from_state(self._isession.state, ds_name, extension, labels_enabled=True)

nexinfosys/ie_exports/flows_graph.py

@@ -7,7 +7,7 @@
 from nexinfosys import ureg
 from nexinfosys.command_generators.parser_ast_evaluators import ast_to_string
 from nexinfosys.models.musiasem_concepts_helper import find_or_create_observable, find_quantitative_observations
-from nexinfosys.restful_service.serialization import deserialize_state
+from nexinfosys.serialization import deserialize_state
 from nexinfosys.command_generators import parser_field_parsers
 
 """

nexinfosys/ie_exports/processors_graph.py

@@ -263,7 +263,7 @@ def construct_processors_graph(state: State, query: IQueryObjects, filt: Union[s
 
 
 if __name__ == '__main__':
-    from nexinfosys.restful_service.serialization import deserialize_state
+    from nexinfosys.serialization import deserialize_state
 
     # Deserialize previously recorded Soslaires State (WARNING! Execute unit tests to generated the ".serialized" file)
     fname = "/home/rnebot/GoogleDrive/AA_MAGIC/Soslaires.serialized"

nexinfosys/ie_imports/data_sources/eurostat_bulk.py

@@ -14,19 +14,16 @@
 import requests
 import requests_cache
 
+from nexinfosys import get_global_configuration_variable
 from nexinfosys.common.helper import create_dictionary, import_names, Memoize2, translate_case
 from nexinfosys.ie_imports.data_source_manager import IDataSourceManager, filter_dataset_into_dataframe
 from nexinfosys.models.statistical_datasets import DataSource, Database, Dataset, Dimension, CodeList, CodeImmutable
 
 
 def create_estat_request():
-    # from magic_box import app  # APP is this now. Adapt to the current app object!
-    app = import_names("nexinfosys.restful_service", "app")
-    if not app:
-        app = import_names("magic_box", "app")
     # EuroStat datasets
-    if 'CACHE_FILE_LOCATION' in app.config:
-        cache_name = app.config['CACHE_FILE_LOCATION']
+    if get_global_configuration_variable('CACHE_FILE_LOCATION'):
+        cache_name = get_global_configuration_variable('CACHE_FILE_LOCATION')
         print("USER: "+getpass.getuser())
         if not os.path.isdir(cache_name):
             os.makedirs(cache_name)

nexinfosys/ie_imports/data_sources/fadn.py

@@ -4,6 +4,9 @@
 import zipfile
 from typing import List
 import pandas as pd
+
+from nexinfosys import get_global_configuration_variable
+
 pd.core.common.is_list_like = pd.api.types.is_list_like
 import pandas_datareader.data as web
 import numpy as np
@@ -82,12 +85,9 @@ def generate_datasets(seed_datasets):
 
 
 def get_fadn_directory():
-    app = import_names("nexinfosys.restful_service", "app")
-    if not app:
-        app = import_names("magic_box", "app")
     # EuroStat datasets
-    if 'FADN_FILES_LOCATION' in app.config:
-        dir_name = app.config['FADN_FILES_LOCATION']
+    if get_global_configuration_variable('FADN_FILES_LOCATION'):
+        dir_name = get_global_configuration_variable('FADN_FILES_LOCATION')
         print("USER: "+getpass.getuser())
     else:
         dir_name = tempfile.gettempdir() + "/fadn_datasets"

nexinfosys/initialization.py

@@ -308,6 +308,8 @@ def get_dataset_from_state(state: State, name: str, extension: str, labels_enabl
             print("Generating Excel")
             ds2.to_excel(output, sheet_name=name, index=False)  # , engine="xlsxwriter")
             return output.getvalue(), "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", True
+        else:
+            return ds2, "application/pandas.dataframe", True
     else:
         return {"error": f"Could not find a Dataset with name '{name}' in the current state"}, "text/json", False
 

requirements-as-package.txt

@@ -0,0 +1,47 @@
+scikit-build>=0.10.0
+appdirs==1.4.3
+toposort>=1.5
+google-api-python-client==1.7.11
+google-auth==1.6.3
+google-auth-httplib2==0.0.3
+google-auth-oauthlib>=0.4.0
+brightway2==2.3
+matplotlib>=3.0.3
+#psycopg2==2.7.3.2
+webdavclient==1.0.8
+owlready2==0.23
+celery>=4.3.0
+pykml==0.1.3
+geopandas==0.4.1
+geojson>=2.4.1
+nbformat>=4.4.0
+typing>=3.6.2
+attrs>=18.1.0
+requests==2.21.0  # >=
+requests_cache==0.4.13
+SQLAlchemy>=1.3.3
+pyparsing>=2.2.0
+numpy>=1.16.0
+pandas==1.0.3
+pandas_datareader>=0.8.1
+pyarrow==1.0.0
+pandaSDMX==0.9
+sdmx>=0.2.10
+regex>=2017.11.9
+chardet>=3.0.4
+aadict>=0.2.3
+anytree>=2.2.2
+networkx==2.2
+multidict>=3.3.2
+xmltodict>=0.11.0
+cubes==1.1
+Pint>=0.8.1
+uncertainties>=3.1.2
+xlrd==1.1.0
+openpyxl==2.4.8
+xlsxwriter==1.0.4
+lxml==4.3.3
+jsonpickle==1.2
+python_magic>=0.4.13  # To be removed, it is used only by the Magic Box file type detection
+dotted>=0.1.8 # <- Dotted notation: !!Fantastic!!
+sympy>=1.1.1

requirements.txt

@@ -10,8 +10,7 @@ gunicorn==19.9.0
 Cython==0.29.7
 # WINDOWS: REMOVE and USE "conda install -c conda-forge python-blosc" (install "gitforwindows", then open terminal using "Run as Administrator")
 blosc>=1.8.1
-# Commented because no LCA dataset was made available during the lifetime of the project, found no way to integrate with this powerful LCA package
-# brightway2==2.3
+brightway2==2.3
 matplotlib>=3.0.3
 #psycopg2==2.7.3.2  # Removed because it requires having PostgreSQL installed. It is explicitly in the Dockerfile
 webdavclient==1.0.8

setup.py

@@ -23,18 +23,20 @@
 # PyPI     : pip install --upgrade nexinfosys
 # No PyPI  : pip install -e <local path where "setup.py" (this file) is located>
 #
-# EXECUTE (example. "gunicorn" must be installed: "pip install gunicorn")
-# (IT WORKS WITH ONLY 1 WORKER!!!)
+# EXECUTION EXAMPLE ("gunicorn" must be installed: "pip install gunicorn")
+#
 # gunicorn --workers=1 --log-level=debug --timeout=2000 --bind 0.0.0.0:8081 nexinfosys.restful_service.service_main:app
 #
+from os import path
+
 from setuptools import setup
 from pkg_resources import yield_lines
 # from distutils.extension import Extension
 from Cython.Build import cythonize
 # from Cython.Distutils import build_ext
 
 package_name = 'nexinfosys'
-version = '0.36'
+version = '0.40'
 
 
 def parse_requirements(strs):
@@ -62,7 +64,7 @@ def parse_requirements(strs):
     return ret
 
 
-with open('requirements.txt') as f:
+with open('requirements-as-package.txt') as f:
     required = f.read().splitlines()
 
 install_reqs = parse_requirements(required)
@@ -73,14 +75,18 @@ def parse_requirements(strs):
 #     Extension("parser_spreadsheet_utils_accel", ["nexinfosys/command_generators/parser_spreadsheet_utils_accel.pyx"])
 # ]
 
+this_directory = path.abspath(path.dirname(__file__))
+with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f:
+    long_description = f.read()
+
+
 setup(
     name=package_name,
     version=version,
     install_requires=install_reqs,
     packages=['nexinfosys', 'nexinfosys.common', 'nexinfosys.models', 'nexinfosys.models.experiments', 'nexinfosys.solving',
               'nexinfosys.solving.graph', 'nexinfosys.ie_exports', 'nexinfosys.ie_imports', 'nexinfosys.ie_imports.data_sources',
               'nexinfosys.ie_imports.experimental', 'nexinfosys.authentication', 'nexinfosys.model_services',
-              'nexinfosys.restful_service', 'nexinfosys.restful_service.gunicorn', 'nexinfosys.restful_service.mod_wsgi',
               'nexinfosys.command_executors', 'nexinfosys.command_executors.misc', 'nexinfosys.command_executors.solving',
               'nexinfosys.command_executors.analysis', 'nexinfosys.command_executors.version2',
               'nexinfosys.command_executors.read_query', 'nexinfosys.command_executors.external_data',
@@ -89,13 +95,18 @@ def parse_requirements(strs):
               'nexinfosys.command_generators.spreadsheet_command_parsers.analysis',
               'nexinfosys.command_generators.spreadsheet_command_parsers.external_data',
               'nexinfosys.command_generators.spreadsheet_command_parsers.specification',
-              'nexinfosys.command_generators.spreadsheet_command_parsers_v2', 'nexinfosys.magic_specific_integrations'],
+              # 'nexinfosys.magic_specific_integrations',
+              'nexinfosys.command_generators.spreadsheet_command_parsers_v2',
+              ],
+    # See files to pack in "MANIFEST.in" file ("frontend" currently disabled)
     include_package_data=True,
     # cmdclass={'build_ext': build_ext},
-    ext_modules=cythonize(["nexinfosys/common/helper_accel.pyx", "nexinfosys/command_generators/parser_spreadsheet_utils_accel.pyx"], language_level="3"),
+    # ext_modules=cythonize(["nexinfosys/common/helper_accel.pyx", "nexinfosys/command_generators/parser_spreadsheet_utils_accel.pyx"], language_level="3"),
     url='https://github.com/MAGIC-nexus/nis-backend',
     license='BSD-3',
-    author='rnebot',
+    author=['Rafael Nebot', 'Marco Galluzzi'],
     author_email='[email protected]',
+    long_description=long_description,
+    long_description_content_type='text/markdown',
     description='Formal and executable MuSIASEM multi-system Nexus models for Sustainable Development Analysis'
 )