Merge pull request #189 from markcoletti/main

Bringing over current ensemble code from forked repo

.gitignore

@@ -1,138 +1,90 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
+/tmp.*
+/local
 *.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-doc/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
+*.pyc
+__pycache__
+docker-nom/build/
+\#*\#
+.\#*
 venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
+*.egg-info/
+docs/build/
+.coverage*
+htmlcov/
+.ipynb_checkpoints/
+.pytest_cache
+build/
+dist/
 
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
+# Subversion
+.svn
 
-# Pyre type checker
-.pyre/
 
-# pytype static type analyzer
-.pytype/
+# Vim (and some others)
+*~
+*.swp
 
-# Cython debug symbols
-cython_debug/
+# Eclipse
+.cache
+.classpath
+.project
+.settings
+build-eclipse
+
+# Gradle
+.gradle
+gradle-app.setting
+
+# Sublime Text
+*.sublime-workspace
+
+# NetBeans
+.netbeans
+catalog.xml
+generated
+nb-configuration.xml
+
+# Mac OS
+.DS_Store
+__MACOSX
+
+# Intellij
+.idea/
+.idea/workspace.xml
+.idea/libraries
+.idea/kotlinc.xml
+.idea/tasks.xml
+.idea/dictionaries
+.idea/scopes
+.idea/artifacts
+.idea/uiDesigner.xml
+.idea/dataSources.local.xml
+.idea/dataSources.local.xml
+.idea/dataSources.ids
+.idea/dataSources.xml
+.idea/dataSources
+.idea/kotlinc.xml
+.idea/sonar*
+*.iws
+*.iml
+.idea/compiler.xml
+.idea/kotlinc.xml
+.idea/inspectionProfiles/profiles_settings.xml
+.idea/misc.xml
+.idea/modules.xml
+
+# vscode
+.vscode
+
+# dask
+dask-worker-space/
+
+# Generated data
+*.csv
+*.png
+
+# Project specific
+simulation_log/
+simulation_setup/
+run.log
+resource_usage

doc/examples/ensembles/a_sim_comp.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+""" Component wrapper for the ensemble example for `a_sim`. """
+from ipsframework import Component
+
+
+class a_sim_comp(Component):
+    def __init__(self, services, config):
+        super().__init__(services, config)
+        print('Created %s' % (self.__class__))
+
+    def step(self, timestamp=0.0):
+        # TODO echo parameters for this run
+        print('Hello from a_sim_comp')
+
+        # Echo the parameters we're expecting, A, B, and C
+        print(f'a_sim_comp parameters: A={self.A}, B={self.B}, C={self.C}')

doc/examples/ensembles/another_sim_comp.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+""" Component wrapper for the ensemble example for `another_sim`. """
+from ipsframework import Component
+
+
+class another_sim_comp(Component):
+    def __init__(self, services, config):
+        super().__init__(services, config)
+        print('Created %s' % (self.__class__))
+
+    def step(self, timestamp=0.0):
+        # TODO echo parameters for this run
+        print('Hello from another_sim_comp')
+
+        # Echo the parameters we're expecting, D, B, and F
+        print(f'another_sim_comp parameters: D={self.D}, B={self.B}, F={self.F}')

doc/examples/ensembles/driver.config

@@ -0,0 +1,31 @@
+# Example configuration file for ensemble support
+
+SIM_ROOT = $PWD
+SIM_NAME = ensemble_example
+ENSEMBLE_DIR = /tmp/ensembles
+ENSEMBLE_PREFIX = MY_INSTANCE_
+LOG_FILE = run.log
+LOG_LEVEL = INFO
+SIMULATION_MODE = NORMAL
+
+[PORTS]
+    NAMES = DRIVER
+
+    [[DRIVER]]
+        IMPLEMENTATION = driver
+
+[driver]
+    # SRC_DIR specified in platform config
+    BIN_PATH = $SRC_DIR/IPS-framework/doc/examples/ensembles/
+    CLASS = driver
+    SUB_CLASS =
+    NAME = ensemble_driver
+    SCRIPT = ${BIN_PATH}/driver.py
+    TEMPLATE = ${BIN_PATH}/template.config
+    PLATFORM_CONFIG_FILE = ${BIN_PATH}/mac135909.config
+    NPROC = 1
+    INPUT_FILES =
+    OUTPUT_FILES =
+    RESTART_FILES =
+
+

doc/examples/ensembles/driver.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""
+    Example driver for the ensembles example.
+
+    Please note that run_ensemble can be run from any IPS component, not
+    just a driver. The driver is used here for simplicity.
+"""
+from pathlib import Path
+
+from ipsframework import Component
+
+
+class ensemble_driver(Component):
+
+    def __init__(self, services, config):
+        super().__init__(services, config)
+        print('Creating driver')
+
+    def init(self, timeStamp=0.0):
+        return
+
+    def step(self, timestamp=0.0, **keywords):
+        """ set up and run the ensemble
+        """
+        # ENSEMBLE_DIR is an arbitrary variable denoting where we want to run
+        # all the ensembles.  You don't have to use ENSEMBLE_DIR. You can
+        # even hardcode the path here.  Whatever.
+        run_dir = Path(self.services.get_config_param('ENSEMBLE_DIR'))
+        self.services.info(f'Running ensemble in {run_dir}')
+
+        # ENSEMBLE_PREFIX is an optional config variable for specifying
+        # a string to prepend to the ensemble run directories and generated
+        # files.  If not specified, the default is 'INSTANCE_n' where n is
+        # is the arbitrary instance ID.
+        prefix = self.services.get_config_param('ENSEMBLE_PREFIX',
+                                                silent=True)
+        if not prefix or prefix == '' or prefix == {}:
+            self.services.info('No ensemble instance prefix specified. Using '
+                               'default of INSTANCE_.')
+            prefix = "INSTANCE_"
+        else:
+            self.services.info(f'Using ensemble instance prefix {prefix}')
+
+        if not run_dir.exists():
+            self.services.info(f'Creating {run_dir}')
+            run_dir.mkdir(parents=True)
+
+        template = Path(self.config['TEMPLATE'])
+        self.services.info(f'Using template config file {template}')
+
+        if not template.exists():
+            raise RuntimeError(f'{template} config template file does not exist')
+
+        # PLATFORM_CONFIG_FILE is a variable that points to the platform
+        # config file used by the ensembles.  This could be the same
+        # platform config file used for this driver, but it doesn't have to be.
+        platform_config = self.config['PLATFORM_CONFIG_FILE']
+        self.services.info(f'Using platform config file {platform_config}')
+
+        # Specifies different sets of variable values for concurrent ensemble
+        # runs for two different components, 'a_sim_comp' and
+        # 'another_sim_comp', that correspond to two different coupled
+        # simulations.  We chose two components to demonstrate that the same
+        # variable, in this case 'B', can have different values for different
+        # components. 'a_sim_comp' and 'another_sim_comp' are the names of
+        # the config sections in the template file so we know where to look
+        # for variable substitutions.
+        variables = {'a_sim_comp': {'A': [3, 2, 4],
+                                    'B': [2.34, 5.82, 0.1],
+                                    'C': ['bar', 'baz', 'quux']},
+                     'another_sim_comp': {'D': [7, 5, 9],
+                                          'B': [0.775, 0.080, 29.2],
+                                          'F': ['xyzzy', 'plud', 'thud']}}
+
+        # Spins up N tasks, in this case three, each with a different set of
+        # variable values. `mapping` is a data struct that associates the
+        # specific simulation to a given run directory so that the user can
+        # easily find output for a specific run.
+        mapping = self.services.run_ensemble(template,
+                                             variables,
+                                             run_dir,
+                                             platform_config,
+                                             prefix)
+
+        self.services.info(f'Mapping of dirs to parameters: {mapping!s}')
+
+
+    def finalize(self, timeStamp=0.0):
+        return
+