From 47844477fdebc45df74f9a397c515c3d563ec7d8 Mon Sep 17 00:00:00 2001 From: Rafael Nebot Medina Date: Tue, 8 Dec 2020 10:09:27 +0000 Subject: [PATCH] Changes to support "pip install nexinfosys" * Reduced number of packages. Specially, avoid use of Flask ("app" variable should not be used outside of "restful_service" package) * "frontend" not included * "setup.py" updated --- .gitignore | 1 + MANIFEST.in | 2 +- README.md | 15 +++--- nexinfosys/__init__.py | 14 ++++++ nexinfosys/embedded_nis.py | 2 + nexinfosys/ie_exports/flows_graph.py | 2 +- nexinfosys/ie_exports/processors_graph.py | 2 +- .../ie_imports/data_sources/eurostat_bulk.py | 9 ++-- nexinfosys/ie_imports/data_sources/fadn.py | 10 ++-- nexinfosys/initialization.py | 2 + requirements-as-package.txt | 47 +++++++++++++++++++ requirements.txt | 3 +- setup.py | 27 +++++++---- 13 files changed, 103 insertions(+), 33 deletions(-) create mode 100644 requirements-as-package.txt diff --git a/.gitignore b/.gitignore index 2d9143c..9f35d6b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ build/ dist/ nexinfosys_backend.egg-info/ +nexinfosys.egg-info/ dump\.rdb nexinfosys/restful_service/nis_local.conf nexinfosys/restful_service/nis_docker_protagoras.conf diff --git a/MANIFEST.in b/MANIFEST.in index ef3478a..84966b2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -recursive-include nexinfosys/frontend * +# recursive-include nexinfosys/frontend */ exclude MANIFEST.in include requirements.txt include nexinfosys/restful_service/nis_local_dist.conf diff --git a/README.md b/README.md index 818464e..d940db3 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ It is an open system as it can be used to integrate MuSIASEM as a formalism in a - [Getting started](#getting-started) - [Features](#features) - [Installing and executing **nis-backend**](#installing-and-executing-nis-backend) + - [Pip package](#pip-package) - [Docker image](#docker-image) - [Source code](#source-code) - [Models with Commands](#models-with-commands) @@ -62,7 +63,7 @@ It is an open system as it can be used to integrate MuSIASEM as a formalism in a - Sources, so several observations can be made about the same fact. - Times, in years or in months. - **Outputs**. Exportable as datasets (statistical cubes), graphs (networks), matrices, models (transformations of the input model), geolayers, scripts (Python or R). -- **Deployment**. It can be deployed in multiple ways: personal computer (installer or pip), or server (Docker or pip). +- **Deployment**. It can be deployed in multiple ways: personal computer (pip, source, Docker), or server (source or Docker). - **Configurable**. Deployment for server can be configured to use different database servers and REDIS for session management. - **Open**. All functionality can be accessed through a RESTful service. Behind the scenes, because HTTP protocol is stateless, a complex serialization/deserialization function combined with a key/value store and a browser session enable saving the state in memory after a service call ends, just to recover it when another invocation is done. - **Two expertise levels**. Two components wrap the RESTful interface, **nis-frontend** and **nis-client**. These components match two expertise levels: **nis-frontend** does not require programming knowledge, while **nis-client** can be used in Python/R scripts. @@ -71,20 +72,16 @@ It is an open system as it can be used to integrate MuSIASEM as a formalism in a **nis-backend** is a Python package. There are several options to have it installed. - +#### pip package - +The pip version is obviously for execution as package, and does not include the frontend, the other two deployment options below include the frontend. - +* Use class class *NIS*, with methods similar to *NISClient* class in [nexinfosys-client](https://github.com/MAGIC-nexus/nis-python-client) package. #### Docker image diff --git a/nexinfosys/__init__.py b/nexinfosys/__init__.py index 41378cb..6549e70 100644 --- a/nexinfosys/__init__.py +++ b/nexinfosys/__init__.py @@ -132,6 +132,7 @@ def default_directories(path, tmp_path): # Flask Session (server side session) REDIS_HOST="filesystem:local_session" TESTING="True" +ENABLE_CYTHON_OPTIMIZATIONS="True" SELF_SCHEMA="" FS_TYPE="WebDAV" FS_SERVER="" @@ -204,6 +205,19 @@ def get_global_configuration_variable(key: str, default: str = None) -> str: return global_configuration.get(key.lower(), default) +def set_global_configuration_variable(key: str, value: str): + """ + Use carefully. The initial intention is to set the "ENABLE_CYTHON_OPTIMIZATIONS" variable to False + + :param key: + :param value: + :return: + """ + global global_configuration + if global_configuration: + global_configuration[key.lower()] = value + + def remove_quotes(s: str) -> str: return s.lstrip('\'"').rstrip('\'"') diff --git a/nexinfosys/embedded_nis.py b/nexinfosys/embedded_nis.py index 0cd4762..48d8ba9 100644 --- a/nexinfosys/embedded_nis.py +++ b/nexinfosys/embedded_nis.py @@ -406,6 +406,8 @@ def get_results(self, datasets: List[Tuple[str, str, str]]): pos = ds_name.find(".") extension = ds_name[pos + 1:] ds_name = ds_name[:pos] + else: + extension = None if struc_type == "dataset": ds, ctype, ok = get_dataset_from_state(self._isession.state, ds_name, extension, labels_enabled=True) diff --git a/nexinfosys/ie_exports/flows_graph.py b/nexinfosys/ie_exports/flows_graph.py index e2b74ff..e7be7c3 100644 --- a/nexinfosys/ie_exports/flows_graph.py +++ b/nexinfosys/ie_exports/flows_graph.py @@ -7,7 +7,7 @@ from nexinfosys import ureg from nexinfosys.command_generators.parser_ast_evaluators import ast_to_string from nexinfosys.models.musiasem_concepts_helper import find_or_create_observable, find_quantitative_observations -from nexinfosys.restful_service.serialization import deserialize_state +from nexinfosys.serialization import deserialize_state from nexinfosys.command_generators import parser_field_parsers """ diff --git a/nexinfosys/ie_exports/processors_graph.py b/nexinfosys/ie_exports/processors_graph.py index 236e458..7892a57 100644 --- a/nexinfosys/ie_exports/processors_graph.py +++ b/nexinfosys/ie_exports/processors_graph.py @@ -263,7 +263,7 @@ def construct_processors_graph(state: State, query: IQueryObjects, filt: Union[s if __name__ == '__main__': - from nexinfosys.restful_service.serialization import deserialize_state + from nexinfosys.serialization import deserialize_state # Deserialize previously recorded Soslaires State (WARNING! Execute unit tests to generated the ".serialized" file) fname = "/home/rnebot/GoogleDrive/AA_MAGIC/Soslaires.serialized" diff --git a/nexinfosys/ie_imports/data_sources/eurostat_bulk.py b/nexinfosys/ie_imports/data_sources/eurostat_bulk.py index 587f688..8c9e04d 100644 --- a/nexinfosys/ie_imports/data_sources/eurostat_bulk.py +++ b/nexinfosys/ie_imports/data_sources/eurostat_bulk.py @@ -14,19 +14,16 @@ import requests import requests_cache +from nexinfosys import get_global_configuration_variable from nexinfosys.common.helper import create_dictionary, import_names, Memoize2, translate_case from nexinfosys.ie_imports.data_source_manager import IDataSourceManager, filter_dataset_into_dataframe from nexinfosys.models.statistical_datasets import DataSource, Database, Dataset, Dimension, CodeList, CodeImmutable def create_estat_request(): - # from magic_box import app # APP is this now. Adapt to the current app object! - app = import_names("nexinfosys.restful_service", "app") - if not app: - app = import_names("magic_box", "app") # EuroStat datasets - if 'CACHE_FILE_LOCATION' in app.config: - cache_name = app.config['CACHE_FILE_LOCATION'] + if get_global_configuration_variable('CACHE_FILE_LOCATION'): + cache_name = get_global_configuration_variable('CACHE_FILE_LOCATION') print("USER: "+getpass.getuser()) if not os.path.isdir(cache_name): os.makedirs(cache_name) diff --git a/nexinfosys/ie_imports/data_sources/fadn.py b/nexinfosys/ie_imports/data_sources/fadn.py index 2afdf7d..7015bd5 100644 --- a/nexinfosys/ie_imports/data_sources/fadn.py +++ b/nexinfosys/ie_imports/data_sources/fadn.py @@ -4,6 +4,9 @@ import zipfile from typing import List import pandas as pd + +from nexinfosys import get_global_configuration_variable + pd.core.common.is_list_like = pd.api.types.is_list_like import pandas_datareader.data as web import numpy as np @@ -82,12 +85,9 @@ def generate_datasets(seed_datasets): def get_fadn_directory(): - app = import_names("nexinfosys.restful_service", "app") - if not app: - app = import_names("magic_box", "app") # EuroStat datasets - if 'FADN_FILES_LOCATION' in app.config: - dir_name = app.config['FADN_FILES_LOCATION'] + if get_global_configuration_variable('FADN_FILES_LOCATION'): + dir_name = get_global_configuration_variable('FADN_FILES_LOCATION') print("USER: "+getpass.getuser()) else: dir_name = tempfile.gettempdir() + "/fadn_datasets" diff --git a/nexinfosys/initialization.py b/nexinfosys/initialization.py index b77473d..979e6cb 100644 --- a/nexinfosys/initialization.py +++ b/nexinfosys/initialization.py @@ -308,6 +308,8 @@ def get_dataset_from_state(state: State, name: str, extension: str, labels_enabl print("Generating Excel") ds2.to_excel(output, sheet_name=name, index=False) # , engine="xlsxwriter") return output.getvalue(), "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", True + else: + return ds2, "application/pandas.dataframe", True else: return {"error": f"Could not find a Dataset with name '{name}' in the current state"}, "text/json", False diff --git a/requirements-as-package.txt b/requirements-as-package.txt new file mode 100644 index 0000000..02749a5 --- /dev/null +++ b/requirements-as-package.txt @@ -0,0 +1,47 @@ +scikit-build>=0.10.0 +appdirs==1.4.3 +toposort>=1.5 +google-api-python-client==1.7.11 +google-auth==1.6.3 +google-auth-httplib2==0.0.3 +google-auth-oauthlib>=0.4.0 +brightway2==2.3 +matplotlib>=3.0.3 +#psycopg2==2.7.3.2 +webdavclient==1.0.8 +owlready2==0.23 +celery>=4.3.0 +pykml==0.1.3 +geopandas==0.4.1 +geojson>=2.4.1 +nbformat>=4.4.0 +typing>=3.6.2 +attrs>=18.1.0 +requests==2.21.0 # >= +requests_cache==0.4.13 +SQLAlchemy>=1.3.3 +pyparsing>=2.2.0 +numpy>=1.16.0 +pandas==1.0.3 +pandas_datareader>=0.8.1 +pyarrow==1.0.0 +pandaSDMX==0.9 +sdmx>=0.2.10 +regex>=2017.11.9 +chardet>=3.0.4 +aadict>=0.2.3 +anytree>=2.2.2 +networkx==2.2 +multidict>=3.3.2 +xmltodict>=0.11.0 +cubes==1.1 +Pint>=0.8.1 +uncertainties>=3.1.2 +xlrd==1.1.0 +openpyxl==2.4.8 +xlsxwriter==1.0.4 +lxml==4.3.3 +jsonpickle==1.2 +python_magic>=0.4.13 # To be removed, it is used only by the Magic Box file type detection +dotted>=0.1.8 # <- Dotted notation: !!Fantastic!! +sympy>=1.1.1 diff --git a/requirements.txt b/requirements.txt index 39fb2ff..93d875d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,8 +10,7 @@ gunicorn==19.9.0 Cython==0.29.7 # WINDOWS: REMOVE and USE "conda install -c conda-forge python-blosc" (install "gitforwindows", then open terminal using "Run as Administrator") blosc>=1.8.1 -# Commented because no LCA dataset was made available during the lifetime of the project, found no way to integrate with this powerful LCA package -# brightway2==2.3 +brightway2==2.3 matplotlib>=3.0.3 #psycopg2==2.7.3.2 # Removed because it requires having PostgreSQL installed. It is explicitly in the Dockerfile webdavclient==1.0.8 diff --git a/setup.py b/setup.py index 2f85b7c..8c4ae5b 100644 --- a/setup.py +++ b/setup.py @@ -23,10 +23,12 @@ # PyPI : pip install --upgrade nexinfosys # No PyPI : pip install -e # -# EXECUTE (example. "gunicorn" must be installed: "pip install gunicorn") -# (IT WORKS WITH ONLY 1 WORKER!!!) +# EXECUTION EXAMPLE ("gunicorn" must be installed: "pip install gunicorn") +# # gunicorn --workers=1 --log-level=debug --timeout=2000 --bind 0.0.0.0:8081 nexinfosys.restful_service.service_main:app # +from os import path + from setuptools import setup from pkg_resources import yield_lines # from distutils.extension import Extension @@ -34,7 +36,7 @@ # from Cython.Distutils import build_ext package_name = 'nexinfosys' -version = '0.36' +version = '0.40' def parse_requirements(strs): @@ -62,7 +64,7 @@ def parse_requirements(strs): return ret -with open('requirements.txt') as f: +with open('requirements-as-package.txt') as f: required = f.read().splitlines() install_reqs = parse_requirements(required) @@ -73,6 +75,11 @@ def parse_requirements(strs): # Extension("parser_spreadsheet_utils_accel", ["nexinfosys/command_generators/parser_spreadsheet_utils_accel.pyx"]) # ] +this_directory = path.abspath(path.dirname(__file__)) +with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + + setup( name=package_name, version=version, @@ -80,7 +87,6 @@ def parse_requirements(strs): packages=['nexinfosys', 'nexinfosys.common', 'nexinfosys.models', 'nexinfosys.models.experiments', 'nexinfosys.solving', 'nexinfosys.solving.graph', 'nexinfosys.ie_exports', 'nexinfosys.ie_imports', 'nexinfosys.ie_imports.data_sources', 'nexinfosys.ie_imports.experimental', 'nexinfosys.authentication', 'nexinfosys.model_services', - 'nexinfosys.restful_service', 'nexinfosys.restful_service.gunicorn', 'nexinfosys.restful_service.mod_wsgi', 'nexinfosys.command_executors', 'nexinfosys.command_executors.misc', 'nexinfosys.command_executors.solving', 'nexinfosys.command_executors.analysis', 'nexinfosys.command_executors.version2', 'nexinfosys.command_executors.read_query', 'nexinfosys.command_executors.external_data', @@ -89,13 +95,18 @@ def parse_requirements(strs): 'nexinfosys.command_generators.spreadsheet_command_parsers.analysis', 'nexinfosys.command_generators.spreadsheet_command_parsers.external_data', 'nexinfosys.command_generators.spreadsheet_command_parsers.specification', - 'nexinfosys.command_generators.spreadsheet_command_parsers_v2', 'nexinfosys.magic_specific_integrations'], + # 'nexinfosys.magic_specific_integrations', + 'nexinfosys.command_generators.spreadsheet_command_parsers_v2', + ], + # See files to pack in "MANIFEST.in" file ("frontend" currently disabled) include_package_data=True, # cmdclass={'build_ext': build_ext}, - ext_modules=cythonize(["nexinfosys/common/helper_accel.pyx", "nexinfosys/command_generators/parser_spreadsheet_utils_accel.pyx"], language_level="3"), + # ext_modules=cythonize(["nexinfosys/common/helper_accel.pyx", "nexinfosys/command_generators/parser_spreadsheet_utils_accel.pyx"], language_level="3"), url='https://github.com/MAGIC-nexus/nis-backend', license='BSD-3', - author='rnebot', + author=['Rafael Nebot', 'Marco Galluzzi'], author_email='rnebot@itccanarias.org', + long_description=long_description, + long_description_content_type='text/markdown', description='Formal and executable MuSIASEM multi-system Nexus models for Sustainable Development Analysis' )