Skip to content

Commit

Permalink
fix examples 004, 005, and 006
Browse files Browse the repository at this point in the history
Signed-off-by: Lance-Drane <[email protected]>
  • Loading branch information
Lance-Drane committed Dec 12, 2024
1 parent f00a2b4 commit 1c3fdf7
Show file tree
Hide file tree
Showing 13 changed files with 51 additions and 61 deletions.
12 changes: 7 additions & 5 deletions examples-proposed/004-jupyter-append/README.md
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
# Task pool synchronous
# Jupyter Notebook appending workflow

This is an example which utilizes the time loop. The config file specifies that the script will execute all timestamps in the range of 1.0 to 100.0 (both inclusive), incrementing by 1.0 for each cycle.

For each timestep, the worker component will update the state file with random JSON data. Note that with this implementation, the state file is overridden on each new timestep (the final timestep called will persist after the application).

Additionally, the `Monitor` component in `component_monitor.py` showcases an example of integrating an IPS run with the IPS JupyterHub workflow. It requires a connection to the IPS web portal to be open to work, but the associated notebooks will show up on the portal. The JupyterHub workflow is designed to allow users to run custom visualizations during an IPS run; the framework can update the data in such a way that it will not interrupt the visualization process.

## Instructions

Note that this example uses the module syntax, as opposed to the script syntax.
Expand All @@ -22,10 +24,10 @@ To run the code, run:
./run.sh
```

By default, this example will always _append_ a state file. If you prefer to see an example of how to _replace_ a state file, run:
By default, this example will always _append_ a state file.

```bash
EXAMPLE_REPLACE=1 ./run.sh
```
The output will generally look like `

There is also a script `run-delayed.sh` which you can use instead of `run.sh` if you would like to simulate a delay between monitor steps.

If executing `run.sh`, the script will output files into the directory defined by the `PSCRATCH` environment variable (or `/tmp` if this variable is not defined).
3 changes: 2 additions & 1 deletion examples-proposed/004-jupyter-append/component_driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

DELAY = bool(os.environ.get('EXAMPLE_DELAY'))


class Driver(Component):
"""In this example, the driver iterates through the time loop and calls both the worker and the monitor component on each timestep."""

Expand All @@ -31,4 +32,4 @@ def step(self, timestamp=0.0):

def finalize(self, timestamp=0.0):
self.services.call(self.worker, 'finalize', 0)
self.services.call(self.monitor, 'finalize', 0)
self.services.call(self.monitor, 'finalize', 0)
15 changes: 5 additions & 10 deletions examples-proposed/004-jupyter-append/component_monitor.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import json
import os
from sys import stderr

from ipsframework import Component
Expand All @@ -17,10 +16,7 @@ def init(self, timestamp=0.0):

# Example of initializing two separate notebooks
# Both notebooks should be initialized before the time loop and appended to inside the time loop
self.services.initialize_jupyter_notebook(
dest_notebook_name='basic.ipynb', # path is relative to JupyterHub directory
source_notebook_path='basic.ipynb', # path is relative to input directory
)
self.services.initialize_jupyter_notebook(NOTEBOOK_1_TEMPLATE)

def step(self, timestamp=0.0, **keywords):
msg = f'Running Monitor step with timestamp={timestamp}'
Expand All @@ -34,18 +30,18 @@ def step(self, timestamp=0.0, **keywords):
# generate any analysis files from the state file you want
with open(state_file) as f:
analysis = json.load(f)

# Do any preprocessing needed prior to adding any file to Jupyter
# NOTE: you must make sure every analysis file name is unique in the "append" workflow

# with the first analysis file, we just dump the results unmodified
analysis_file_1 = os.path.join(self.services.get_config_param('SIM_ROOT'), f'{timestamp}_analysis.json')
analysis_file_1 = f'{timestamp}_analysis.json'
with open(analysis_file_1, 'w') as f:
json.dump(analysis, f)

# with the second analysis file, we'll just do a mock "analysis" where we just divide the values by two
mapped_analysis = {key: value / 2 for key, value in analysis.items()}
analysis_file_2 = os.path.join(self.services.get_config_param('SIM_ROOT'), f'{timestamp}_analysis_mapped.json')
analysis_file_2 = f'{timestamp}_analysis_mapped.json'
with open(analysis_file_2, 'w') as f:
json.dump(mapped_analysis, f)

Expand All @@ -60,5 +56,4 @@ def step(self, timestamp=0.0, **keywords):
print('SEND PORTAL DATA', timestamp, raw_data, file=stderr)
self.services.send_portal_data(timestamp, raw_data)

def finalize(self, timestamp=0.0):
...
def finalize(self, timestamp=0.0): ...
4 changes: 2 additions & 2 deletions examples-proposed/004-jupyter-append/component_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ def step(self, timestamp=0.0):
print(msg, file=stderr)
self.services.send_portal_event(event_comment=msg)

# TODO - maybe make y2 and y3 slightly more unique with formulas (don't just multiply)
data = {
'y1': math.sin(self.start + timestamp / 50 * math.pi),
'y2': math.sin(self.start + timestamp / 50 * math.pi) ** 2,
Expand All @@ -31,5 +32,4 @@ def step(self, timestamp=0.0):
json.dump(data, f)
self.services.update_state()

def finalize(self, timestamp=0.0):
...
def finalize(self, timestamp=0.0): ...
10 changes: 5 additions & 5 deletions examples-proposed/005-jupyter-replace/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ This is an example which utilizes the time loop. The config file specifies that

For each timestep, the worker component will update the state file with random JSON data. Note that with this implementation, the state file is overridden on each new timestep (the final timestep called will persist after the application).

Additionally, the `Monitor` component in `component_monitor.py` showcases an example of integrating an IPS run with the IPS JupyterHub workflow. It requires a connection to the IPS web portal to be open to work, but the associated notebooks will show up on the portal. The JupyterHub workflow is designed to allow users to run custom visualizations during an IPS run; the framework can update the data in such a way that it will not interrupt the visualization process.

## Instructions

Note that this example uses the module syntax, as opposed to the script syntax.
Expand All @@ -22,10 +24,8 @@ To run the code, run:
./run.sh
```

By default, this example will always _append_ a state file. If you prefer to see an example of how to _replace_ a state file, run:

```bash
EXAMPLE_REPLACE=1 ./run.sh
```
By default, this example will always _replace_ a state file.

There is also a script `run-delayed.sh` which you can use instead of `run.sh` if you would like to simulate a delay between monitor steps.

If executing `run.sh`, the script will output files into the directory defined by the `PSCRATCH` environment variable (or `/tmp` if this variable is not defined).
3 changes: 2 additions & 1 deletion examples-proposed/005-jupyter-replace/component_driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

DELAY = bool(os.environ.get('EXAMPLE_DELAY'))


class Driver(Component):
"""In this example, the driver iterates through the time loop and calls both the worker and the monitor component on each timestep."""

Expand All @@ -31,4 +32,4 @@ def step(self, timestamp=0.0):

def finalize(self, timestamp=0.0):
self.services.call(self.worker, 'finalize', 0)
self.services.call(self.monitor, 'finalize', 0)
self.services.call(self.monitor, 'finalize', 0)
11 changes: 3 additions & 8 deletions examples-proposed/005-jupyter-replace/component_monitor.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import json
import os
from sys import stderr

from ipsframework import Component
Expand All @@ -20,10 +19,7 @@ def init(self, timestamp=0.0):

# Example of initializing two separate notebooks
# Both notebooks should be initialized before the time loop and appended to inside the time loop
self.services.initialize_jupyter_notebook(
dest_notebook_name='basic.ipynb', # path is relative to JupyterHub directory
source_notebook_path='basic.ipynb', # path is relative to input directory
)
self.services.initialize_jupyter_notebook(NOTEBOOK_1_TEMPLATE)

def step(self, timestamp=0.0, **keywords):
msg = f'Running Monitor step with timestamp={timestamp}'
Expand All @@ -38,7 +34,7 @@ def step(self, timestamp=0.0, **keywords):
with open(state_file) as f:
analysis = json.load(f)

analysis_file_1 = os.path.join(self.services.get_config_param('SIM_ROOT'), f'{timestamp}_analysis.json')
analysis_file_1 = f'{timestamp}_analysis.json'
with open(analysis_file_1, 'w') as f:
json.dump(analysis, f)

Expand All @@ -48,5 +44,4 @@ def step(self, timestamp=0.0, **keywords):
print('SEND PORTAL DATA', timestamp, raw_data, file=stderr)
self.services.send_portal_data(timestamp, raw_data)

def finalize(self, timestamp=0.0):
...
def finalize(self, timestamp=0.0): ...
3 changes: 1 addition & 2 deletions examples-proposed/005-jupyter-replace/component_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,5 +31,4 @@ def step(self, timestamp=0.0):
json.dump(data, f)
self.services.update_state()

def finalize(self, timestamp=0.0):
...
def finalize(self, timestamp=0.0): ...
4 changes: 4 additions & 0 deletions examples-proposed/006-jupyter-multiple-notebooks/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ This is an example which utilizes the time loop. The config file specifies that

For each timestep, the worker component will update the state file with random JSON data. Note that with this implementation, the state file is overridden on each new timestep (the final timestep called will persist after the application).

Additionally, the `Monitor` component in `component_monitor.py` showcases an example of integrating an IPS run with the IPS JupyterHub workflow. It requires a connection to the IPS web portal to be open to work, but the associated notebooks will show up on the portal. The JupyterHub workflow is designed to allow users to run custom visualizations during an IPS run; the framework can update the data in such a way that it will not interrupt the visualization process.

## Instructions

Note that this example uses the module syntax, as opposed to the script syntax.
Expand All @@ -29,3 +31,5 @@ EXAMPLE_REPLACE=1 ./run.sh
```

There is also a script `run-delayed.sh` which you can use instead of `run.sh` if you would like to simulate a delay between monitor steps.

If executing `run.sh`, the script will output files into the directory defined by the `PSCRATCH` environment variable (or `/tmp` if this variable is not defined).
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

DELAY = bool(os.environ.get('EXAMPLE_DELAY'))


class Driver(Component):
"""In this example, the driver iterates through the time loop and calls both the worker and the monitor component on each timestep."""

Expand All @@ -31,4 +32,4 @@ def step(self, timestamp=0.0):

def finalize(self, timestamp=0.0):
self.services.call(self.worker, 'finalize', 0)
self.services.call(self.monitor, 'finalize', 0)
self.services.call(self.monitor, 'finalize', 0)
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import json
import os
from sys import stderr

from ipsframework import Component
Expand All @@ -21,14 +20,8 @@ def init(self, timestamp=0.0):

# Example of initializing two separate notebooks
# Both notebooks should be initialized before the time loop and appended to inside the time loop
self.services.initialize_jupyter_notebook(
dest_notebook_name=NOTEBOOK_1_TEMPLATE, # path is relative to JupyterHub directory
source_notebook_path=NOTEBOOK_1_TEMPLATE, # path is relative to input directory
)
self.services.initialize_jupyter_notebook(
dest_notebook_name=NOTEBOOK_2_TEMPLATE, # path is relative to JupyterHub directory
source_notebook_path=NOTEBOOK_2_TEMPLATE, # path is relative to input directory
)
self.services.initialize_jupyter_notebook(NOTEBOOK_1_TEMPLATE)
self.services.initialize_jupyter_notebook(NOTEBOOK_2_TEMPLATE)

def step(self, timestamp=0.0, **keywords):
msg = f'Running Monitor step with timestamp={timestamp}'
Expand All @@ -47,7 +40,7 @@ def step(self, timestamp=0.0, **keywords):
with open(state_file) as f:
analysis = json.load(f)

analysis_file_1 = os.path.join(self.services.get_config_param('SIM_ROOT'), f'{timestamp}_analysis.json')
analysis_file_1 = f'{timestamp}_analysis.json'
with open(analysis_file_1, 'w') as f:
json.dump(analysis, f)
data = json.dumps(analysis).encode()
Expand All @@ -60,5 +53,4 @@ def step(self, timestamp=0.0, **keywords):
print('SEND PORTAL DATA', timestamp, data, file=stderr)
self.services.send_portal_data(timestamp, data)

def finalize(self, timestamp=0.0):
...
def finalize(self, timestamp=0.0): ...
Original file line number Diff line number Diff line change
Expand Up @@ -31,5 +31,4 @@ def step(self, timestamp=0.0):
json.dump(data, f)
self.services.update_state()

def finalize(self, timestamp=0.0):
...
def finalize(self, timestamp=0.0): ...
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,12 @@
"\n",
"output_notebook()\n",
"\n",
"DATA = []\n",
"# create DATA list, will depend on user input type (i.e. 'hdf5', 'json')\n",
"for file in DATA_FILES.values():\n",
" with open(file, 'rb') as f:\n",
" DATA.append(json.load(f))\n",
"x = list(DATA_FILES.keys())\n",
"# get data function - will depend on user input (i.e. 'hdf5', 'json', etc.)\n",
"def get_data(d, prop_path):\n",
" data = d[prop_path[0]]\n",
" for inner in prop_path[1:]:\n",
" data = data[inner]\n",
" return data\n",
"\n",
"COLORS = ['red', 'green', 'blue']\n",
"\n",
Expand All @@ -33,12 +33,13 @@
" {'title': 'IPS partial results', 'graph_format': 'line', 'data_paths': [['y1']]},\n",
"]\n",
"\n",
"# get data function - will depend on user input (i.e. 'hdf5', 'json', etc.)\n",
"def get_data(d, prop_path):\n",
" data = d[prop_path[0]]\n",
" for inner in prop_path[1:]:\n",
" data = data[inner]\n",
" return data\n",
"x = list(DATA_FILES.keys())\n",
"DATA = []\n",
"for filelist in DATA_FILES.values():\n",
" # create DATA list, will depend on user input type (i.e. 'hdf5', 'json')\n",
" for file in filelist:\n",
" with open(file, 'rb') as f:\n",
" DATA.append(json.load(f))\n",
"\n",
"# generate two simple line graphs from data\n",
"for g in GRAPHS:\n",
Expand Down

0 comments on commit 1c3fdf7

Please sign in to comment.