Refactor "backend" to "nexinfosys"; other features

* Renamed "backend" package to "nexinfosys", so distribution with "pip" does a better deployment.
* Also, related to deployment with "pip", "frontend" directory has been moved inside "nexinfosys".
* When executing (parsing and solving) a NIS file, when an exception occurs, capture it and return a single Issue.
* Deployment with setup.py.
* Deployment with Docker modified.

.gitignore

@@ -2,19 +2,24 @@
 .idea
 .mypy_cache
 
+build/
+dist/
+nexinfosys_backend.egg-info/
 dump\.rdb
-backend/restful_service/nis_local.conf
-backend/restful_service/nis_docker_protagoras.conf
-backend/restful_service/nis_local_sqlite.conf
-backend/restful_service/nis_docker_local.conf
-backend/restful_service/nis_docker_naples.conf
+nexinfosys/restful_service/nis_local.conf
+nexinfosys/restful_service/nis_docker_protagoras.conf
+nexinfosys/restful_service/nis_local_sqlite.conf
+nexinfosys/restful_service/nis_docker_local.conf
+nexinfosys/restful_service/nis_docker_naples.conf
 install.sh
 Input Output entites of each command.gdsheet
 strategy.ini
-backend/ie_imports/experimental/UN Comtrade Country List.csv
-backend/ie_imports/experimental/country_alternative_names.csv
-backend/ie_imports/experimental/saved_queries.csv
-backend/ie_imports/experimental/0.csv
+nexinfosys/ie_imports/experimental/UN Comtrade Country List.csv
+nexinfosys/ie_imports/experimental/country_alternative_names.csv
+nexinfosys/ie_imports/experimental/saved_queries.csv
+nexinfosys/ie_imports/experimental/0.csv
 magic_box/
-/backend/restful_service/credentials.json
-/backend/restful_service/token.pickle
+/nexinfosys/restful_service/credentials.json
+/nexinfosys/restful_service/token.pickle
+
+~/nis_data/fadn/measures.csv

Dockerfile

@@ -13,7 +13,7 @@ FROM python:3.7.3-slim-stretch
 #
 # docker start nis-local && docker logs nis-local -f
 #
-# LOCAL SERVER: (remember to start a REDIS instance) the image would be:
+# LOCAL SERVER: (the configuration uses REDISLITE, so no need to have a REDIS instance) the image would be:
 #
 # docker create --name nis-local -p 8080:80
 #               -v /home/rnebot/DATOS/docker/nis-local:/srv
@@ -88,8 +88,7 @@ RUN pip3 install --no-cache-dir psycopg2==2.7.3.2 && \
 
 COPY supervisord.conf /etc/supervisord.conf
 
-COPY backend /app/backend
-COPY frontend /app/frontend
+COPY nexinfosys /app/nexinfosys
 RUN mkdir -p /srv
 
 EXPOSE 80
@@ -99,5 +98,5 @@ VOLUME /srv
 ENV C_FORCE_ROOT=1
 
 # run supervisord
-CMD ["/usr/local/bin/gunicorn", "--workers=3", "--log-level=debug", "--timeout=2000", "--bind", "0.0.0.0:80", "backend.restful_service.service_main:app"]
+CMD ["/usr/local/bin/gunicorn", "--workers=1", "--log-level=debug", "--timeout=2000", "--bind", "0.0.0.0:80", "nexinfosys.restful_service.service_main:app"]
 #CMD ["supervisord", "-c", "/etc/supervisord.conf"]

MANIFEST.in

@@ -1 +1,5 @@
-recursive-include frontend *
+recursive-include nexinfosys/frontend *
+exclude MANIFEST.in
+include requirements.txt
+include nexinfosys/restful_service/nis_local_dist.conf
+exclude nexinfosys/restful_service/nis_docker_naples.conf

backend_tests/test_command_container_files_execution.py

@@ -1,21 +1,21 @@
 import os
 import unittest
 
-import backend
-from backend.ie_exports.json import export_model_to_json
-from backend.model_services import get_case_study_registry_objects
-from backend.model_services.workspace import execute_file, prepare_and_reset_database_for_tests, \
+import nexinfosys
+from nexinfosys.ie_exports.json import export_model_to_json
+from nexinfosys.model_services import get_case_study_registry_objects
+from nexinfosys.model_services.workspace import execute_file, prepare_and_reset_database_for_tests, \
     prepare_and_solve_model, execute_file_return_issues
-from backend.models.musiasem_concepts import Observer, \
+from nexinfosys.models.musiasem_concepts import Observer, \
     Processor, FactorType, Factor, \
     Hierarchy, \
     FactorQuantitativeObservation, ProcessorsRelationPartOfObservation, \
     ProcessorsRelationUndirectedFlowObservation, ProcessorsRelationUpscaleObservation, \
     FactorsRelationDirectedFlowObservation
 # Database (ORM)
-from backend.restful_service import register_external_datasources
-from backend.restful_service.serialization import serialize_state, deserialize_state
-from backend.solving import get_processor_names_to_processors_dictionary
+from nexinfosys.restful_service import register_external_datasources
+from nexinfosys.restful_service.serialization import serialize_state, deserialize_state
+from nexinfosys.solving import get_processor_names_to_processors_dictionary
 
 
 class TestFAOCommandFiles(unittest.TestCase):
@@ -26,7 +26,7 @@ def setUpClass(cls):
             prepare=True,
             metadata_string="sqlite:////home/rnebot/GoogleDrive/AA_MAGIC/nis_metadata.db",
             data_string="sqlite:////home/rnebot/GoogleDrive/AA_MAGIC/nis_cached_datasets.db")
-        backend.data_source_manager = register_external_datasources(
+        nexinfosys.data_source_manager = register_external_datasources(
             {"FAO_DATASETS_DIR": "/home/marco/temp/Data/FAOSTAT/"})
 
     @classmethod
@@ -60,7 +60,7 @@ class TestCommandFiles(unittest.TestCase):
     def setUpClass(cls):
         # Executed BEFORE test methods of the class
         prepare_and_reset_database_for_tests(prepare=True)
-        backend.data_source_manager = register_external_datasources(
+        nexinfosys.data_source_manager = register_external_datasources(
             {"FAO_DATASETS_DIR": "/home/marco/temp/Data/FAOSTAT/"})
 
     @classmethod
@@ -786,7 +786,7 @@ def test_024_export_JupyterNotebook(self):
     #
     # a = 1 / 0
 
-    from backend.common.helper import add_label_columns_to_dataframe
+    from nexinfosys.common.helper import add_label_columns_to_dataframe
 
     is_fao_test = False
     fao_dir = "/home/marco/temp/Data/FAOSTAT/"
@@ -796,13 +796,13 @@ def test_024_export_JupyterNotebook(self):
             prepare=True,
             metadata_string="sqlite:////home/rnebot/GoogleDrive/AA_MAGIC/nis_metadata.db",
             data_string="sqlite:////home/rnebot/GoogleDrive/AA_MAGIC/nis_cached_datasets.db")
-        backend.data_source_manager = register_external_datasources(
+        nexinfosys.data_source_manager = register_external_datasources(
             {"FAO_DATASETS_DIR": fao_dir})
         i.test_001_fao()
     else:
         i = TestCommandFiles()
         prepare_and_reset_database_for_tests(prepare=True)
-        backend.data_source_manager = register_external_datasources(
+        nexinfosys.data_source_manager = register_external_datasources(
             {"FAO_DATASETS_DIR": fao_dir})
 
         #i.test_002_execute_file_two()

backend_tests/test_integration_use_cases.py

@@ -2,33 +2,33 @@
 import sqlalchemy
 
 # Memory
-from backend.model_services.workspace import InteractiveSession, CreateNew, prepare_and_reset_database_for_tests
-from backend.command_executors import create_command
-from backend.restful_service import tm_default_users, \
+from nexinfosys.model_services.workspace import InteractiveSession, CreateNew, prepare_and_reset_database_for_tests
+from nexinfosys.command_executors import create_command
+from nexinfosys.restful_service import tm_default_users, \
     tm_authenticators, \
     tm_object_types, \
     tm_permissions, \
     tm_case_study_version_statuses
 
 # Database (ORM)
-from backend.models.musiasem_methodology_support import *
-import backend
+from nexinfosys.models.musiasem_methodology_support import *
+import nexinfosys
 
 
 def setUpModule():
     print('In setUpModule()')
     # Setup SQLAlchemy engines as SQLite IN-MEMORY
-    backend.engine = sqlalchemy.create_engine("sqlite://", echo=True)
-    backend.data_engine = sqlalchemy.create_engine("sqlite://", echo=True)
+    nexinfosys.engine = sqlalchemy.create_engine("sqlite://", echo=True)
+    nexinfosys.data_engine = sqlalchemy.create_engine("sqlite://", echo=True)
 
     # global DBSession # global DBSession registry to get the scoped_session
-    DBSession.configure(bind=backend.engine)  # reconfigure the sessionmaker used by this scoped_session
+    DBSession.configure(bind=nexinfosys.engine)  # reconfigure the sessionmaker used by this scoped_session
     tables = ORMBase.metadata.tables
-    connection = backend.engine.connect()
-    table_existence = [backend.engine.dialect.has_table(connection, tables[t].name) for t in tables]
+    connection = nexinfosys.engine.connect()
+    table_existence = [nexinfosys.engine.dialect.has_table(connection, tables[t].name) for t in tables]
     connection.close()
     if False in table_existence:
-        ORMBase.metadata.bind = backend.engine
+        ORMBase.metadata.bind = nexinfosys.engine
         ORMBase.metadata.create_all()
 
     # Load base tables
@@ -52,8 +52,8 @@ def setUpModule():
 
 def tearDownModule():
     print('In tearDownModule()')
-    backend.data_engine.dispose()
-    backend.engine.dispose()
+    nexinfosys.data_engine.dispose()
+    nexinfosys.engine.dispose()
     print(str(len(tm_authenticators)))
     print("a")
 

backend_tests/test_integration_web_service.py

@@ -25,13 +25,13 @@
  
 """
 
-import backend.restful_service  # Just to remove error in the "importlib.reload" sentence below
+import nexinfosys.restful_service  # Just to remove error in the "importlib.reload" sentence below
 os.environ["MAGIC_NIS_SERVICE_CONFIG_FILE"] = "../../../nis-backend-config/nis_unittests.conf"
-if "backend.restful_service" in sys.modules:
-    importlib.reload(backend.restful_service)
-backend.restful_service.app.config["TESTING"] = "True"
-import backend.restful_service.service_main
-from backend.restful_service import app, nis_api_base
+if "nexinfosys.restful_service" in sys.modules:
+    importlib.reload(nexinfosys.restful_service)
+nexinfosys.restful_service.app.config["TESTING"] = "True"
+import nexinfosys.restful_service.service_main
+from nexinfosys.restful_service import app, nis_api_base
 
 
 def to_str(resp_data):

backend_tests/test_unit_computation_graph.py

@@ -1,9 +1,9 @@
 import unittest
 from typing import List, Set, Dict, Tuple, Optional, Callable
 
-from backend.solving.graph.computation_graph import ComputationGraph
-from backend.solving.graph.flow_graph import FlowGraph
-from backend.solving.graph import EdgeType
+from nexinfosys.solving.graph.computation_graph import ComputationGraph
+from nexinfosys.solving.graph.flow_graph import FlowGraph
+from nexinfosys.solving.graph import EdgeType
 
 
 class SubTestCase:

backend_tests/test_unit_helper.py

@@ -4,19 +4,19 @@
 
 pyximport.install(reload_support=True, language_level=3)
 
-from backend.common.helper_accel import augment_dataframe_with_mapped_columns2
-import backend.common.helper
-from backend.common.helper import PartialRetrievalDictionary, augment_dataframe_with_mapped_columns, create_dictionary
-from backend.models.musiasem_concepts import Processor, ProcessorsRelationPartOfObservation, Observer
-from backend.models.musiasem_methodology_support import (
+from nexinfosys.common.helper_accel import augment_dataframe_with_mapped_columns2
+import nexinfosys.common.helper
+from nexinfosys.common.helper import PartialRetrievalDictionary, augment_dataframe_with_mapped_columns, create_dictionary
+from nexinfosys.models.musiasem_concepts import Processor, ProcessorsRelationPartOfObservation, Observer
+from nexinfosys.models.musiasem_methodology_support import (
                                                       serialize_from_object,
                                                       deserialize_to_object
                                                       )
 
 
 def setUpModule():
     print('In setUpModule()')
-    backend.common.helper.case_sensitive = True
+    nexinfosys.common.helper.case_sensitive = True
 
 
 def tearDownModule():
@@ -253,7 +253,7 @@ def test_003_delete(self):
         self.assertEqual(len(res), 0)
 
     def test_004_case_insensitive(self):
-        backend.common.helper.case_sensitive = False
+        nexinfosys.common.helper.case_sensitive = False
         prd = prepare_partial_key_dictionary()
         # Tests
         res = prd.get({"_type": "partOf"})

backend_tests/test_unit_musiasem_concepts.py

@@ -1,12 +1,12 @@
 import unittest
 
 
-import backend.common.helper
-from backend.models.experiments.expressions import ExpressionsEngine
-from backend.models.musiasem_concepts import *
-from backend.models.musiasem_concepts_helper import *
-from backend.models.musiasem_concepts_helper import _get_observer, _find_or_create_relation
-from backend.restful_service.serialization import serialize_state, deserialize_state
+import nexinfosys.common.helper
+from nexinfosys.models.experiments.expressions import ExpressionsEngine
+from nexinfosys.models.musiasem_concepts import *
+from nexinfosys.models.musiasem_concepts_helper import *
+from nexinfosys.models.musiasem_concepts_helper import _get_observer, _find_or_create_relation
+from nexinfosys.restful_service.serialization import serialize_state, deserialize_state
 
 """ Integration tests for in memory model structures """
 
@@ -269,7 +269,7 @@ def tearDown(self):
         super().tearDown()
 
     def test_001_build_soslaires_relations(self):
-        backend.common.helper.case_sensitive = False
+        nexinfosys.common.helper.case_sensitive = False
         state = State()
         create_relation_observations(state, "WindFarm:WindElectricity", ["DesalinationPlant", ("ElectricGrid")])
         create_relation_observations(state, "ElectricGrid", "DesalinationPlant:GridElectricity")

backend/authentication/__init__.py → nexinfosys/authentication/__init__.py

@@ -8,7 +8,7 @@
 """
 from abc import ABCMeta, abstractmethod
 
-from backend.common.helper import create_dictionary
+from nexinfosys.common.helper import create_dictionary
 
 
 class IAuthenticator(metaclass=ABCMeta):