From 6a6f0b8b23b38d2506d5da4c1598d94b128275ab Mon Sep 17 00:00:00 2001 From: guzzijones Date: Fri, 7 Jul 2023 21:09:16 +0000 Subject: [PATCH 01/32] remove some deepcopy to speed up workflow conductor --- orquesta/conducting.py | 46 +++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/orquesta/conducting.py b/orquesta/conducting.py index 34fb3567..18d7634a 100644 --- a/orquesta/conducting.py +++ b/orquesta/conducting.py @@ -49,12 +49,12 @@ def __init__(self, conductor=None): def serialize(self): data = { - "contexts": json_util.deepcopy(self.contexts), - "routes": json_util.deepcopy(self.routes), - "sequence": json_util.deepcopy(self.sequence), + "contexts": self.contexts, + "routes": self.routes, + "sequence": self.sequence, "staged": json_util.deepcopy(self.staged), "status": self.status, - "tasks": json_util.deepcopy(self.tasks), + "tasks": self.tasks, } if self.reruns: @@ -65,13 +65,13 @@ def serialize(self): @classmethod def deserialize(cls, data): instance = cls() - instance.contexts = json_util.deepcopy(data.get("contexts", list())) - instance.routes = json_util.deepcopy(data.get("routes", list())) - instance.sequence = json_util.deepcopy(data.get("sequence", list())) - instance.staged = json_util.deepcopy(data.get("staged", list())) + instance.contexts = data.get("contexts", list()) + instance.routes = data.get("routes", list()) + instance.sequence = data.get("sequence", list()) + instance.staged = data.get("staged", list()) instance.status = data.get("status", statuses.UNSET) - instance.tasks = json_util.deepcopy(data.get("tasks", dict())) - instance.reruns = json_util.deepcopy(data.get("reruns", list())) + instance.tasks = data.get("tasks", dict()) + instance.reruns = data.get("reruns", list()) return instance @@ -281,8 +281,8 @@ def serialize(self): "input": self.get_workflow_input(), "context": self.get_workflow_parent_context(), "state": self.workflow_state.serialize(), - "log": json_util.deepcopy(self.log), - "errors": json_util.deepcopy(self.errors), + "log": self.log, + "errors": self.errors, "output": self.get_workflow_output(), } @@ -292,12 +292,12 @@ def deserialize(cls, data): spec = spec_module.WorkflowSpec.deserialize(data["spec"]) graph = graphing.WorkflowGraph.deserialize(data["graph"]) - inputs = json_util.deepcopy(data["input"]) - context = json_util.deepcopy(data["context"]) + inputs = data["input"] + context = data["context"] state = WorkflowState.deserialize(data["state"]) - log = json_util.deepcopy(data.get("log", [])) - errors = json_util.deepcopy(data["errors"]) - outputs = json_util.deepcopy(data["output"]) + log = data.get("log", []) + errors = data["errors"] + outputs = data["output"] instance = cls(spec) instance.restore(graph, log, errors, state, inputs, outputs, context) @@ -412,7 +412,7 @@ def get_workflow_parent_context(self): return json_util.deepcopy(self._parent_ctx) def get_workflow_input(self): - return json_util.deepcopy(self._inputs) + return self._inputs def get_workflow_status(self): return self.workflow_state.status @@ -461,7 +461,7 @@ def request_workflow_status(self, status): raise exc.InvalidWorkflowStatusTransition(current_status, wf_ex_event.name) def get_workflow_initial_context(self): - return json_util.deepcopy(self.workflow_state.contexts[0]) + return self.workflow_state.contexts[0] def get_workflow_terminal_context(self): if self.get_workflow_status() not in statuses.COMPLETED_STATUSES: @@ -513,7 +513,7 @@ def render_workflow_output(self): self.request_workflow_status(statuses.FAILED) def get_workflow_output(self): - return json_util.deepcopy(self._outputs) if self._outputs else None + return self._outputs if self._outputs else None def reset_workflow_output(self): self._outputs = None @@ -780,7 +780,7 @@ def setup_retry_in_task_state(self, task_state_entry, in_ctx_idxs): # Setup the retry in the task state. task_id = task_state_entry["id"] task_retry_spec = self.graph.get_task_retry_spec(task_id) - task_state_entry["retry"] = json_util.deepcopy(task_retry_spec) + task_state_entry["retry"] = task_retry_spec task_state_entry["retry"]["tally"] = 0 # Get task context for evaluating the expression in delay and count. @@ -1186,8 +1186,8 @@ def get_task_transition_contexts(self, task_id, route): def _request_task_rerun(self, task_id, route, reset_items=False): task = self.workflow_state.get_task(task_id, route) - task_ctx = json_util.deepcopy(task["ctxs"]["in"]) - task_prev = json_util.deepcopy(task["prev"]) + task_ctx = task["ctxs"]["in"] + task_prev = task["prev"] task_spec = self.spec.tasks.get_task(task_id) # Reset terminal status for the rerunnable candidate. From 0f5d37484eda334a4d81c4052854db58d511fd4c Mon Sep 17 00:00:00 2001 From: AJ Date: Sat, 8 Jul 2023 00:46:43 +0000 Subject: [PATCH 02/32] Update tox.yml remove 3.6 tox for now --- .github/workflows/tox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 55ddce3f..1cbfcb15 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6", "3.8"] + python-version: ["3.8"] steps: - name: Checkout repository uses: actions/checkout@v2 From 73522e4a5d91c30067b0c20befe4ac1bdb712f6d Mon Sep 17 00:00:00 2001 From: AJ Date: Sat, 8 Jul 2023 22:55:29 +0000 Subject: [PATCH 03/32] Update tox.yml set ubuntu-20.04 --- .github/workflows/tox.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 1cbfcb15..749cc273 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -12,10 +12,10 @@ on: jobs: tests: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.8"] + python-version: ["3.6", "3.8"] steps: - name: Checkout repository uses: actions/checkout@v2 From 46c7546f965984022ec1dd9b57048630cc980744 Mon Sep 17 00:00:00 2001 From: AJ Date: Sat, 8 Jul 2023 23:19:29 +0000 Subject: [PATCH 04/32] Update tox.yml stevedore is not working in the python 3,8 unit tests --- .github/workflows/tox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 749cc273..ac301d91 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.6", "3.8"] + python-version: ["3.6"] steps: - name: Checkout repository uses: actions/checkout@v2 From 3bff3e0dcd4c0651759b778ccfbd3c279087dbd8 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 10 Jul 2023 13:46:58 +0000 Subject: [PATCH 05/32] black formatting --- orquesta/conducting.py | 1 - orquesta/tests/hacking/import_aliases_rule.py | 2 -- orquesta/tests/unit/base.py | 2 -- .../unit/conducting/test_workflow_conductor_data_flow.py | 5 ++--- orquesta/tests/unit/utils/test_strings.py | 2 +- 5 files changed, 3 insertions(+), 9 deletions(-) diff --git a/orquesta/conducting.py b/orquesta/conducting.py index 18d7634a..13196aeb 100644 --- a/orquesta/conducting.py +++ b/orquesta/conducting.py @@ -370,7 +370,6 @@ def log_entry( result=None, data=None, ): - # Check entry type. if entry_type not in ["info", "warn", "error"]: raise exc.WorkflowLogEntryError('The log entry type "%s" is not valid.' % entry_type) diff --git a/orquesta/tests/hacking/import_aliases_rule.py b/orquesta/tests/hacking/import_aliases_rule.py index f6eb2539..8114bb58 100644 --- a/orquesta/tests/hacking/import_aliases_rule.py +++ b/orquesta/tests/hacking/import_aliases_rule.py @@ -59,7 +59,6 @@ def get_alias(logical_line): - parts = logical_line.split() if ( @@ -68,7 +67,6 @@ def get_alias(logical_line): and parts[1] != "__future__" and not core.is_import_exception(parts[1]) ): - # from path.to.module import module if len(parts) == 4: return ".".join([parts[1], parts[3]]), None diff --git a/orquesta/tests/unit/base.py b/orquesta/tests/unit/base.py index a4aa287f..d0cca988 100644 --- a/orquesta/tests/unit/base.py +++ b/orquesta/tests/unit/base.py @@ -156,7 +156,6 @@ def format_task_item( items_count=None, items_concurrency=None, ): - if not actions and items_count is None: actions = [{"action": spec.action, "input": spec.input}] @@ -285,7 +284,6 @@ def assert_task_items( concurrency=None, mock_ac_ex_results=None, ): - # Set up test cases. tests = list(zip(mock_ac_ex_statuses, expected_task_statuses, expected_workflow_statuses)) tk_ex_result = [None] * len(items) diff --git a/orquesta/tests/unit/conducting/test_workflow_conductor_data_flow.py b/orquesta/tests/unit/conducting/test_workflow_conductor_data_flow.py index 02822925..caa8eb07 100644 --- a/orquesta/tests/unit/conducting/test_workflow_conductor_data_flow.py +++ b/orquesta/tests/unit/conducting/test_workflow_conductor_data_flow.py @@ -25,7 +25,6 @@ class WorkflowConductorDataFlowTest(test_base.WorkflowConductorTest): - wf_def_yaql = """ version: 1.0 @@ -157,14 +156,14 @@ def assert_data_flow(self, input_value): def assert_unicode_data_flow(self, input_value): inputs = { - u"a1": ( + "a1": ( str_util.unicode(input_value, encoding_type="utf-8", force=True) if six.PY2 else input_value ) } - expected_output = {u"a5": inputs["a1"], u"b5": inputs["a1"]} + expected_output = {"a5": inputs["a1"], "b5": inputs["a1"]} self._assert_data_flow(inputs, expected_output) diff --git a/orquesta/tests/unit/utils/test_strings.py b/orquesta/tests/unit/utils/test_strings.py index 4817aaa9..0cab37c7 100644 --- a/orquesta/tests/unit/utils/test_strings.py +++ b/orquesta/tests/unit/utils/test_strings.py @@ -28,7 +28,7 @@ def test_unescape(self): def test_unicode(self): self.assertEqual(str_util.unicode(123), 123) self.assertEqual(str_util.unicode("foobar"), "foobar") - self.assertEqual(str_util.unicode(u"fubar" if six.PY2 else str("fubar")), "fubar") + self.assertEqual(str_util.unicode("fubar" if six.PY2 else str("fubar")), "fubar") self.assertEqual(str_util.unicode("鐵甲奇俠"), "鐵甲奇俠") self.assertEqual(str_util.unicode("\xe9\x90\xb5\xe7\x94\xb2"), "\xe9\x90\xb5\xe7\x94\xb2") From 3805276d1d8d879166d66a05d7cb0535a6e0434e Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 10 Jul 2023 14:08:02 +0000 Subject: [PATCH 06/32] add back python 3.8 --- .github/workflows/tox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index ac301d91..749cc273 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.6"] + python-version: ["3.6", "3.8"] steps: - name: Checkout repository uses: actions/checkout@v2 From 8ce4a4290b181d08d17cf6db5ca1e62c48fb71a5 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 10 Jul 2023 14:58:21 +0000 Subject: [PATCH 07/32] try 18.04 --- .github/workflows/tox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 749cc273..cb408311 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -12,7 +12,7 @@ on: jobs: tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-18.04 strategy: matrix: python-version: ["3.6", "3.8"] From 61bf8468cb10e2bde748866f4f1a72f432418f47 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 10 Jul 2023 15:25:17 +0000 Subject: [PATCH 08/32] switch back to 20.04 --- .github/workflows/tox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index cb408311..749cc273 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -12,7 +12,7 @@ on: jobs: tests: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 strategy: matrix: python-version: ["3.6", "3.8"] From bbd2ea4669af01f36cdbbe3385d548a54e7d27ea Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 10 Jul 2023 16:54:06 +0000 Subject: [PATCH 09/32] remove unittest2 --- requirements-test.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-test.txt b/requirements-test.txt index dd206521..eba7eb2c 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -8,4 +8,3 @@ nosexcover pep8>=1.6.0,<1.7 pylint>=2.5.2,<2.6 twine -unittest2 From cb5234860efa701d0635fe5cccda0e65b483bad0 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 10 Jul 2023 16:58:54 +0000 Subject: [PATCH 10/32] install setup.py to load entrypoints --- .github/workflows/tox.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 749cc273..b8aa9225 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -27,6 +27,7 @@ jobs: run: | python -m pip install --upgrade pip pip install tox tox-gh-actions + pip install . - name: "Run tox for ${{ matrix.python-version }}" run: | tox From 7bdb5436fee630abca8591a954d9002ee9e8ea6c Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 10 Jul 2023 17:06:49 +0000 Subject: [PATCH 11/32] set language --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 4c0402d4..0832406b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -60,7 +60,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'English' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. From 8fd8bb1659bdea16df866d51befecca252e64f9d Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 10 Jul 2023 19:28:40 +0000 Subject: [PATCH 12/32] remove another deepcopy --- orquesta/tests/unit/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orquesta/tests/unit/base.py b/orquesta/tests/unit/base.py index d0cca988..94197486 100644 --- a/orquesta/tests/unit/base.py +++ b/orquesta/tests/unit/base.py @@ -217,7 +217,7 @@ def assert_task_list(self, conductor, actual, expected): for task in expected_copy: task["ctx"]["__current_task"] = {"id": task["id"], "route": task["route"]} - task["ctx"]["__state"] = conductor.workflow_state.serialize() + task["ctx"]["__state"] = copy.deepcopy(conductor.workflow_state.serialize()) for staged_task in task["ctx"]["__state"]["staged"]: if "items" in staged_task: From aac13bf03858ad1eaeac0521144d86880b9303ab Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 10 Jul 2023 19:33:41 +0000 Subject: [PATCH 13/32] remove staged copy --- orquesta/conducting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orquesta/conducting.py b/orquesta/conducting.py index 13196aeb..ff97d892 100644 --- a/orquesta/conducting.py +++ b/orquesta/conducting.py @@ -52,7 +52,7 @@ def serialize(self): "contexts": self.contexts, "routes": self.routes, "sequence": self.sequence, - "staged": json_util.deepcopy(self.staged), + "staged": self.staged, "status": self.status, "tasks": self.tasks, } From 4712345754d5a293674edbb7570fb6cac00f5435 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 10 Jul 2023 20:01:10 +0000 Subject: [PATCH 14/32] remove reruns deepcopy --- orquesta/conducting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orquesta/conducting.py b/orquesta/conducting.py index ff97d892..9fc0123c 100644 --- a/orquesta/conducting.py +++ b/orquesta/conducting.py @@ -58,7 +58,7 @@ def serialize(self): } if self.reruns: - data["reruns"] = json_util.deepcopy(self.reruns) + data["reruns"] = self.reruns return data From f5caf3adffbf0be14291e2fb3532639acfe79049 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 10 Jul 2023 21:17:06 +0000 Subject: [PATCH 15/32] remove serialize parent_ctx copy --- orquesta/conducting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orquesta/conducting.py b/orquesta/conducting.py index 9fc0123c..84c5fa2c 100644 --- a/orquesta/conducting.py +++ b/orquesta/conducting.py @@ -279,7 +279,7 @@ def serialize(self): "spec": self.spec.serialize(), "graph": self.graph.serialize(), "input": self.get_workflow_input(), - "context": self.get_workflow_parent_context(), + "context": self._parent_ctx, "state": self.workflow_state.serialize(), "log": self.log, "errors": self.errors, From 6478f9ba2aca19717b067106d194e5d4c8ae6376 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Tue, 11 Jul 2023 15:16:22 +0000 Subject: [PATCH 16/32] deepcopy machine.py, deepcopy graphing.py, conducting.py --- orquesta/conducting.py | 8 ++++---- orquesta/graphing.py | 2 +- orquesta/machines.py | 7 ++++--- orquesta/tests/unit/conducting/test_workflow_conductor.py | 4 ++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/orquesta/conducting.py b/orquesta/conducting.py index 84c5fa2c..21b9b3c9 100644 --- a/orquesta/conducting.py +++ b/orquesta/conducting.py @@ -317,7 +317,7 @@ def workflow_state(self): self._workflow_state = WorkflowState(conductor=self) # Set any given context as the initial context. - init_ctx = self.get_workflow_parent_context() + init_ctx = self.get_workflow_parent_context_copy() # Render workflow inputs and merge into the initial context. workflow_input = self.get_workflow_input() @@ -407,7 +407,7 @@ def log_errors(self, errors, task_id=None, route=None, task_transition_id=None): error, task_id=task_id, route=route, task_transition_id=task_transition_id ) - def get_workflow_parent_context(self): + def get_workflow_parent_context_copy(self): return json_util.deepcopy(self._parent_ctx) def get_workflow_input(self): @@ -481,8 +481,8 @@ def get_workflow_terminal_context(self): for idx, task in other_term_tasks: # Remove the initial context since the first task processed above already # inclulded that and we only want to apply the differences. - in_ctx_idxs = json_util.deepcopy(task["ctxs"]["in"]) - in_ctx_idxs.remove(0) + in_ctx_idxs = [i for index, i in enumerate(task["ctxs"]["in"]) if + index != 0] wf_term_ctx = dict_util.merge_dicts( wf_term_ctx, self.get_task_context(in_ctx_idxs), overwrite=True diff --git a/orquesta/graphing.py b/orquesta/graphing.py index 6c718cb1..4944fd79 100644 --- a/orquesta/graphing.py +++ b/orquesta/graphing.py @@ -47,7 +47,7 @@ def serialize(self): @classmethod def deserialize(cls, data): - g = json_graph.adjacency_graph(json_util.deepcopy(data), directed=True, multigraph=True) + g = json_graph.adjacency_graph(data, directed=True, multigraph=True) return cls(graph=g) @staticmethod diff --git a/orquesta/machines.py b/orquesta/machines.py index 55b41be0..4a59c6aa 100644 --- a/orquesta/machines.py +++ b/orquesta/machines.py @@ -527,11 +527,12 @@ def add_context_to_task_item_event(cls, workflow_state, task_id, task_route, ac_ if ac_ex_event.status in requirements: # Make a copy of the items and remove current item under evaluation. staged_task = workflow_state.get_staged_task(task_id, task_route) - items = json_util.deepcopy(staged_task["items"]) - del items[ac_ex_event.item_id] - items_status = [item.get("status", statuses.UNSET) for item in items] + items = staged_task["items"] + items_status = [item.get("status", statuses.UNSET) for index, item in + enumerate(items) if index != ac_ex_event.item_id] # Assess various situations. + # todo(aj) loop over list one time and add to each list active = list(filter(lambda x: x in statuses.ACTIVE_STATUSES, items_status)) incomplete = list(filter(lambda x: x not in statuses.COMPLETED_STATUSES, items_status)) paused = list(filter(lambda x: x in [statuses.PENDING, statuses.PAUSED], items_status)) diff --git a/orquesta/tests/unit/conducting/test_workflow_conductor.py b/orquesta/tests/unit/conducting/test_workflow_conductor.py index cbbe5ddd..00a80b97 100644 --- a/orquesta/tests/unit/conducting/test_workflow_conductor.py +++ b/orquesta/tests/unit/conducting/test_workflow_conductor.py @@ -97,7 +97,7 @@ def _prep_conductor(self, context=None, inputs=None, status=None): self.assertDictEqual(conductor._inputs, user_inputs) self.assertDictEqual(conductor.get_workflow_input(), user_inputs) self.assertDictEqual(conductor._parent_ctx, parent_context) - self.assertDictEqual(conductor.get_workflow_parent_context(), parent_context) + self.assertDictEqual(conductor.get_workflow_parent_context_copy(), parent_context) default_inputs = {"a": None, "b": False} init_ctx_value = dict_util.merge_dicts(default_inputs, user_inputs, True) @@ -282,7 +282,7 @@ def test_serialization(self): "spec": conductor.spec.serialize(), "graph": conductor.graph.serialize(), "state": conductor.workflow_state.serialize(), - "context": conductor.get_workflow_parent_context(), + "context": conductor.get_workflow_parent_context_copy(), "input": conductor.get_workflow_input(), "output": conductor.get_workflow_output(), "errors": conductor.errors, From 0f58986f7709a9015ac630c698e3fb3b177aa1ad Mon Sep 17 00:00:00 2001 From: guzzijones Date: Tue, 11 Jul 2023 18:05:47 +0000 Subject: [PATCH 17/32] remove json_util --- orquesta/machines.py | 1 - 1 file changed, 1 deletion(-) diff --git a/orquesta/machines.py b/orquesta/machines.py index 4a59c6aa..0f97142c 100644 --- a/orquesta/machines.py +++ b/orquesta/machines.py @@ -18,7 +18,6 @@ from orquesta import events from orquesta import exceptions as exc from orquesta import statuses -from orquesta.utils import jsonify as json_util LOG = logging.getLogger(__name__) From d3541fbabc5d7d774ac414add624ec558009c7a0 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Wed, 12 Jul 2023 23:37:01 +0000 Subject: [PATCH 18/32] need to copy staged so it isn't mutated for with items --- orquesta/conducting.py | 5 ++--- orquesta/machines.py | 7 +++++-- orquesta/tests/unit/base.py | 11 +---------- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/orquesta/conducting.py b/orquesta/conducting.py index 21b9b3c9..11773313 100644 --- a/orquesta/conducting.py +++ b/orquesta/conducting.py @@ -68,7 +68,7 @@ def deserialize(cls, data): instance.contexts = data.get("contexts", list()) instance.routes = data.get("routes", list()) instance.sequence = data.get("sequence", list()) - instance.staged = data.get("staged", list()) + instance.staged = json_util.deepcopy(data.get("staged", list())) instance.status = data.get("status", statuses.UNSET) instance.tasks = data.get("tasks", dict()) instance.reruns = data.get("reruns", list()) @@ -481,8 +481,7 @@ def get_workflow_terminal_context(self): for idx, task in other_term_tasks: # Remove the initial context since the first task processed above already # inclulded that and we only want to apply the differences. - in_ctx_idxs = [i for index, i in enumerate(task["ctxs"]["in"]) if - index != 0] + in_ctx_idxs = [i for index, i in enumerate(task["ctxs"]["in"]) if index != 0] wf_term_ctx = dict_util.merge_dicts( wf_term_ctx, self.get_task_context(in_ctx_idxs), overwrite=True diff --git a/orquesta/machines.py b/orquesta/machines.py index 0f97142c..8657eec6 100644 --- a/orquesta/machines.py +++ b/orquesta/machines.py @@ -527,8 +527,11 @@ def add_context_to_task_item_event(cls, workflow_state, task_id, task_route, ac_ # Make a copy of the items and remove current item under evaluation. staged_task = workflow_state.get_staged_task(task_id, task_route) items = staged_task["items"] - items_status = [item.get("status", statuses.UNSET) for index, item in - enumerate(items) if index != ac_ex_event.item_id] + items_status = [ + item.get("status", statuses.UNSET) + for index, item in enumerate(items) + if index != ac_ex_event.item_id + ] # Assess various situations. # todo(aj) loop over list one time and add to each list diff --git a/orquesta/tests/unit/base.py b/orquesta/tests/unit/base.py index 94197486..3b846cf1 100644 --- a/orquesta/tests/unit/base.py +++ b/orquesta/tests/unit/base.py @@ -209,20 +209,11 @@ def assert_task_list(self, conductor, actual, expected): expected_copy = copy.deepcopy(expected) for task in actual_copy: - for staged_task in task["ctx"]["__state"]["staged"]: - if "items" in staged_task: - del staged_task["items"] - task["spec"] = task["spec"].serialize() for task in expected_copy: task["ctx"]["__current_task"] = {"id": task["id"], "route": task["route"]} - task["ctx"]["__state"] = copy.deepcopy(conductor.workflow_state.serialize()) - - for staged_task in task["ctx"]["__state"]["staged"]: - if "items" in staged_task: - del staged_task["items"] - + task["ctx"]["__state"] = conductor.workflow_state.serialize() task["spec"] = task["spec"].serialize() self.assertListEqual(actual_copy, expected_copy) From 6e3866a0884334dcbbac686f0890291f468446cb Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 24 Jul 2023 15:34:59 +0000 Subject: [PATCH 19/32] ensure staged is deep copied --- .../unit/conducting/test_workflow_state.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/orquesta/tests/unit/conducting/test_workflow_state.py b/orquesta/tests/unit/conducting/test_workflow_state.py index 52909689..5c683b53 100644 --- a/orquesta/tests/unit/conducting/test_workflow_state.py +++ b/orquesta/tests/unit/conducting/test_workflow_state.py @@ -71,6 +71,26 @@ def test_get_tasks_by_task_id(self): self.assertListEqual(actual_task_sequence, expected_task_sequence) + def test_stage_is_deepcopied(self): + # if staged is not deep copied then an st2 workflow with a failed + # with_items task will never finish running. It will remain in a + # running state forever. I believe it is due to iteration over staged + # tasks and mutation of the staged section. + data = copy.deepcopy(MOCK_WORKFLOW_STATE) + + task_sequence = [ + {"id": "task1", "route": 0}, + {"id": "task2", "route": 0}, + {"id": "task2", "route": 1}, + {"id": "task2", "route": 2}, + {"id": "task3", "route": 0}, + ] + + data["sequence"] = copy.deepcopy(task_sequence) + state = conducting.WorkflowState.deserialize(data) + MOCK_WORKFLOW_STATE["staged"]=["something"] + self.assertNotEqual(len(state.staged), len(MOCK_WORKFLOW_STATE["staged"])) + def test_get_tasks_by_task_id_and_route(self): data = copy.deepcopy(MOCK_WORKFLOW_STATE) From 62a23a5dbd851769d7587a38cca9a02605167c15 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Mon, 24 Jul 2023 15:47:32 +0000 Subject: [PATCH 20/32] flake8 lint fix --- orquesta/tests/unit/conducting/test_workflow_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orquesta/tests/unit/conducting/test_workflow_state.py b/orquesta/tests/unit/conducting/test_workflow_state.py index 5c683b53..a038ebcf 100644 --- a/orquesta/tests/unit/conducting/test_workflow_state.py +++ b/orquesta/tests/unit/conducting/test_workflow_state.py @@ -88,7 +88,7 @@ def test_stage_is_deepcopied(self): data["sequence"] = copy.deepcopy(task_sequence) state = conducting.WorkflowState.deserialize(data) - MOCK_WORKFLOW_STATE["staged"]=["something"] + MOCK_WORKFLOW_STATE["staged"] = ["something"] self.assertNotEqual(len(state.staged), len(MOCK_WORKFLOW_STATE["staged"])) def test_get_tasks_by_task_id_and_route(self): From 1364dca91614d343b8d0c8b18d4ebf0bd6bf3fb8 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Thu, 27 Jul 2023 15:33:15 +0000 Subject: [PATCH 21/32] add back deepcopy for errors as they are also mutated in st2 --- orquesta/conducting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orquesta/conducting.py b/orquesta/conducting.py index 11773313..b62f4cc9 100644 --- a/orquesta/conducting.py +++ b/orquesta/conducting.py @@ -296,7 +296,7 @@ def deserialize(cls, data): context = data["context"] state = WorkflowState.deserialize(data["state"]) log = data.get("log", []) - errors = data["errors"] + errors = json_util.deepcopy(data["errors"]) outputs = data["output"] instance = cls(spec) From d9816faa23ef081d99d42dd8960ca0ae66513f90 Mon Sep 17 00:00:00 2001 From: guzzijones Date: Tue, 23 Jan 2024 19:25:59 +0000 Subject: [PATCH 22/32] do not copy with items context; added benchmarks --- .../benchmarks/specs/native/v1/test_models.py | 127 ++++++++++++++++++ orquesta/conducting.py | 1 + orquesta/specs/native/v1/models.py | 20 +-- setup.py | 4 + 4 files changed, 144 insertions(+), 8 deletions(-) create mode 100644 orquesta/benchmarks/specs/native/v1/test_models.py diff --git a/orquesta/benchmarks/specs/native/v1/test_models.py b/orquesta/benchmarks/specs/native/v1/test_models.py new file mode 100644 index 00000000..4c899be5 --- /dev/null +++ b/orquesta/benchmarks/specs/native/v1/test_models.py @@ -0,0 +1,127 @@ +import pytest + +import orquesta.specs.native.v1.models as models + +from orquesta.expressions import base as expr_base +from orquesta.utils import context as ctx_util + + +WITH_ITEMS = [ + [{"test": "s"}, {"test1": "s"}, {"test2": "s"}], + [{"test": "s" * 10000}, {"test1": "s" * 10000}, {"test2": "s" * 10000}], + [{"test": "s" * 1000000}, {"test1": "s" * 1000000}, {"test2": "s" * 1000000}], +] + + +@pytest.mark.parametrize("fixture", WITH_ITEMS, ids=["small", "medium", "large"]) +@pytest.mark.benchmark(group="no deepcopy") +def test_task_spec_render(benchmark, fixture): + def run_benchmark(): + # Instantiate workflow spec. + task_spec = models.TaskSpec( + { + "action": "core.echo message=<% item() %>", + "next": [{"publish": [{"items": "<% result() %>"}]}], + "with": {"items": "<% ctx(xs) %>"}, + } + ) + in_ctx = { + "xs": fixture, + "__current_task": {"id": "task1", "route": 0}, + "__state": { + "contexts": [{"xs": fixture}], + "routes": [[]], + "sequence": [], + "staged": [ + {"id": "task1", "ctxs": {"in": [0]}, "route": 0, "prev": {}, "ready": True} + ], + "status": "running", + "tasks": {}, + }, + } + # Instantiate conductor + res, actions = task_spec.render(in_ctx) + + res = benchmark(run_benchmark) + + +class OldTaskSpec(models.TaskSpec): + def render(self, in_ctx): + action_specs = [] + + if not self.has_items(): + action_spec = { + "action": expr_base.evaluate(self.action, in_ctx), + "input": expr_base.evaluate(getattr(self, "input", {}), in_ctx), + } + + action_specs.append(action_spec) + else: + items_spec = self.get_items_spec() + + if " in " not in items_spec.items: + items_expr = items_spec.items.strip() + else: + start_idx = items_spec.items.index(" in ") + 4 + items_expr = items_spec.items[start_idx:].strip() + + items = expr_base.evaluate(items_expr, in_ctx) + + if not isinstance(items, list): + raise TypeError('The value of "%s" is not type of list.' % items_expr) + + item_keys = ( + None + if " in " not in items_spec.items + else items_spec.items[: items_spec.items.index(" in ")].replace(" ", "").split(",") + ) + + for idx, item in enumerate(items): + if item_keys and (isinstance(item, tuple) or isinstance(item, list)): + item = dict(zip(item_keys, list(item))) + elif item_keys and len(item_keys) == 1: + item = {item_keys[0]: item} + + item_ctx_value = ctx_util.set_current_item(in_ctx, item) + + action_spec = { + "action": expr_base.evaluate(self.action, item_ctx_value), + "input": expr_base.evaluate(getattr(self, "input", {}), item_ctx_value), + "item_id": idx, + } + + action_specs.append(action_spec) + + return self, action_specs + + +@pytest.mark.parametrize("fixture", WITH_ITEMS, ids=["small", "medium", "large"]) +@pytest.mark.benchmark(group="deepcopy") +def test_task_spec_render_old(benchmark, fixture): + def run_benchmark(): + # Instantiate workflow spec. + task_spec = OldTaskSpec( + { + "action": "core.echo message=<% item() %>", + "next": [{"publish": [{"items": "<% result() %>"}]}], + "with": {"items": "<% ctx(xs) %>"}, + } + ) + in_ctx = { + "xs": fixture, + "__current_task": {"id": "task1", "route": 0}, + "__state": { + "contexts": [{"xs": fixture}], + "routes": [[]], + "sequence": [], + "staged": [ + {"id": "task1", "ctxs": {"in": [0]}, "route": 0, "prev": {}, "ready": True} + ], + "status": "running", + "tasks": {}, + }, + } + # Instantiate conductor + res, actions = task_spec.render(in_ctx) + + res = benchmark(run_benchmark) diff --git a/orquesta/conducting.py b/orquesta/conducting.py index b62f4cc9..11d9df84 100644 --- a/orquesta/conducting.py +++ b/orquesta/conducting.py @@ -575,6 +575,7 @@ def get_task(self, task_id, route): task_ctx = ctx_util.set_current_task(task_ctx, current_task) task_ctx = dict_util.merge_dicts(task_ctx, state_ctx, True) task_spec = self.spec.tasks.get_task(task_id).copy() + # here is where taskspec.render is used task_spec, action_specs = task_spec.render(task_ctx) task = { diff --git a/orquesta/specs/native/v1/models.py b/orquesta/specs/native/v1/models.py index 1712dfac..286c4f70 100644 --- a/orquesta/specs/native/v1/models.py +++ b/orquesta/specs/native/v1/models.py @@ -155,7 +155,6 @@ def has_retry(self): def render(self, in_ctx): action_specs = [] - if not self.has_items(): action_spec = { "action": expr_base.evaluate(self.action, in_ctx), @@ -182,21 +181,26 @@ def render(self, in_ctx): if " in " not in items_spec.items else items_spec.items[: items_spec.items.index(" in ")].replace(" ", "").split(",") ) - for idx, item in enumerate(items): if item_keys and (isinstance(item, tuple) or isinstance(item, list)): item = dict(zip(item_keys, list(item))) elif item_keys and len(item_keys) == 1: item = {item_keys[0]: item} - item_ctx_value = ctx_util.set_current_item(in_ctx, item) + if in_ctx and not isinstance(in_ctx, dict): + raise TypeError("The context is not type of dict.") - action_spec = { - "action": expr_base.evaluate(self.action, item_ctx_value), - "input": expr_base.evaluate(getattr(self, "input", {}), item_ctx_value), - "item_id": idx, - } + in_ctx["__current_item"] = item + try: + action_spec = { + "action": expr_base.evaluate(self.action, in_ctx), + "input": expr_base.evaluate(getattr(self, "input", {}), in_ctx), + "item_id": idx, + } + finally: + in_ctx.pop("__current_item", None) + # expr_base.evaluate does the copy for us here. action_specs.append(action_spec) return self, action_specs diff --git a/setup.py b/setup.py index f6fe0179..16659046 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,10 @@ def get_requirements(): packages=find_packages(exclude=[]), install_requires=get_requirements(), license="Apache License (2.0)", + extras_require={ + "tests": ["coverage", "pytest>=6.2.2", "pytest-cov", "mock"], + "build": ["wheel"], + }, classifiers=[ "Development Status :: 4 - Beta", "Intended Audience :: Information Technology", From 016f825076917a99de87887040e094add371b87d Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Tue, 23 Jan 2024 20:28:03 +0000 Subject: [PATCH 23/32] remove comment about task render --- orquesta/conducting.py | 1 - 1 file changed, 1 deletion(-) diff --git a/orquesta/conducting.py b/orquesta/conducting.py index 11d9df84..b62f4cc9 100644 --- a/orquesta/conducting.py +++ b/orquesta/conducting.py @@ -575,7 +575,6 @@ def get_task(self, task_id, route): task_ctx = ctx_util.set_current_task(task_ctx, current_task) task_ctx = dict_util.merge_dicts(task_ctx, state_ctx, True) task_spec = self.spec.tasks.get_task(task_id).copy() - # here is where taskspec.render is used task_spec, action_specs = task_spec.render(task_ctx) task = { From 8cf2ff87e94d1b9babe681ee589db8c8ab66bcaf Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Tue, 23 Jan 2024 20:35:19 +0000 Subject: [PATCH 24/32] add test requirements --- requirements-test.txt | 1 + tox.ini | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements-test.txt b/requirements-test.txt index eba7eb2c..2e6ad107 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -8,3 +8,4 @@ nosexcover pep8>=1.6.0,<1.7 pylint>=2.5.2,<2.6 twine +pytest-benchmark diff --git a/tox.ini b/tox.ini index 40c149af..9601cdea 100644 --- a/tox.ini +++ b/tox.ini @@ -15,6 +15,7 @@ deps = -r{toxinidir}/requirements-test.txt commands = nosetests -sv --with-xcoverage --cover-package=orquesta orquesta.tests + nosetests -sv orquesta.benchmarks [testenv:pep8] deps = From eb098563602f25242f9a701c93c0868ed6ee20eb Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Tue, 23 Jan 2024 20:44:02 +0000 Subject: [PATCH 25/32] typo in benchmark vs benchmarks --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9877be48..cea40124 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ deps = -r{toxinidir}/requirements-test.txt commands = pytest --cov=orquesta --cov-report=term orquesta/tests - pytest orquesta/benchmark + pytest orquesta/benchmarks [testenv:pep8] deps = From c5dbc16526c79faf5d1ed7c7ae5b5834ed639ccb Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Tue, 23 Jan 2024 20:58:06 +0000 Subject: [PATCH 26/32] add __init__.py so imports work --- orquesta/benchmarks/__init__.py | 0 orquesta/benchmarks/specs/__init__.py | 0 orquesta/benchmarks/specs/native/__init__.py | 0 orquesta/benchmarks/specs/native/v1/__init__.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 orquesta/benchmarks/__init__.py create mode 100644 orquesta/benchmarks/specs/__init__.py create mode 100644 orquesta/benchmarks/specs/native/__init__.py create mode 100644 orquesta/benchmarks/specs/native/v1/__init__.py diff --git a/orquesta/benchmarks/__init__.py b/orquesta/benchmarks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orquesta/benchmarks/specs/__init__.py b/orquesta/benchmarks/specs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orquesta/benchmarks/specs/native/__init__.py b/orquesta/benchmarks/specs/native/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orquesta/benchmarks/specs/native/v1/__init__.py b/orquesta/benchmarks/specs/native/v1/__init__.py new file mode 100644 index 00000000..e69de29b From 84a6aa2ce1c7cdda4b7cd45328305f97ee8246bb Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Tue, 23 Jan 2024 21:04:45 +0000 Subject: [PATCH 27/32] remove unused import ctx_util --- orquesta/specs/native/v1/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/orquesta/specs/native/v1/models.py b/orquesta/specs/native/v1/models.py index 286c4f70..30318b1c 100644 --- a/orquesta/specs/native/v1/models.py +++ b/orquesta/specs/native/v1/models.py @@ -22,7 +22,6 @@ from orquesta.expressions import base as expr_base from orquesta.specs.native.v1 import base as native_v1_specs from orquesta.specs import types as spec_types -from orquesta.utils import context as ctx_util from orquesta.utils import dictionary as dict_util from orquesta.utils import jsonify as json_util from orquesta.utils import parameters as args_util From a1f4685d161e801c995f67d1a2d6db5c0b6a68a8 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Tue, 23 Jan 2024 21:11:55 +0000 Subject: [PATCH 28/32] flake8 fixes --- orquesta/benchmarks/specs/native/v1/test_models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/orquesta/benchmarks/specs/native/v1/test_models.py b/orquesta/benchmarks/specs/native/v1/test_models.py index 4c899be5..a31520f0 100644 --- a/orquesta/benchmarks/specs/native/v1/test_models.py +++ b/orquesta/benchmarks/specs/native/v1/test_models.py @@ -40,9 +40,9 @@ def run_benchmark(): }, } # Instantiate conductor - res, actions = task_spec.render(in_ctx) + task_spec.render(in_ctx) - res = benchmark(run_benchmark) + benchmark(run_benchmark) class OldTaskSpec(models.TaskSpec): @@ -122,6 +122,6 @@ def run_benchmark(): }, } # Instantiate conductor - res, actions = task_spec.render(in_ctx) + task_spec.render(in_ctx) - res = benchmark(run_benchmark) + benchmark(run_benchmark) From 3c2db2c464dc3c1476c8e46095dfa62b90d836af Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Tue, 23 Jan 2024 21:21:02 +0000 Subject: [PATCH 29/32] add license file --- .../benchmarks/specs/native/v1/test_models.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/orquesta/benchmarks/specs/native/v1/test_models.py b/orquesta/benchmarks/specs/native/v1/test_models.py index a31520f0..6d429b6b 100644 --- a/orquesta/benchmarks/specs/native/v1/test_models.py +++ b/orquesta/benchmarks/specs/native/v1/test_models.py @@ -1,3 +1,18 @@ +# Copyright 2021 The StackStorm Authors. +# Copyright 2019 Extreme Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import pytest import orquesta.specs.native.v1.models as models From 429f0e06312c6fa5a20a19a01aff81e09c55e41a Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Tue, 23 Jan 2024 21:30:39 +0000 Subject: [PATCH 30/32] remove extras require --- setup.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup.py b/setup.py index 16659046..f6fe0179 100644 --- a/setup.py +++ b/setup.py @@ -54,10 +54,6 @@ def get_requirements(): packages=find_packages(exclude=[]), install_requires=get_requirements(), license="Apache License (2.0)", - extras_require={ - "tests": ["coverage", "pytest>=6.2.2", "pytest-cov", "mock"], - "build": ["wheel"], - }, classifiers=[ "Development Status :: 4 - Beta", "Intended Audience :: Information Technology", From 25cf175ab4dccd542951c653b8a1a7fccf0f2a02 Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Tue, 30 Jan 2024 19:51:42 +0000 Subject: [PATCH 31/32] more deep copy removed --- orquesta/specs/native/v1/models.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/orquesta/specs/native/v1/models.py b/orquesta/specs/native/v1/models.py index 30318b1c..be7f13c4 100644 --- a/orquesta/specs/native/v1/models.py +++ b/orquesta/specs/native/v1/models.py @@ -14,6 +14,7 @@ # limitations under the License. import logging +import copy import six from six.moves import queue @@ -205,7 +206,7 @@ def render(self, in_ctx): return self, action_specs def finalize_context(self, next_task_name, task_transition_meta, in_ctx): - rolling_ctx = json_util.deepcopy(in_ctx) + rolling_ctx = copy.copy(in_ctx) new_ctx = {} errors = [] @@ -640,7 +641,8 @@ def __init__(self, spec, name=None, member=False): super(WorkflowSpec, self).__init__(spec, name=name, member=member) def render_input(self, runtime_inputs, in_ctx=None): - rolling_ctx = json_util.deepcopy(in_ctx) if in_ctx else {} + # only replacing top key values in dict a copy is fine here + rolling_ctx = copy.copy(in_ctx) if in_ctx else {} errors = [] for input_spec in getattr(self, "input") or []: @@ -662,7 +664,8 @@ def render_input(self, runtime_inputs, in_ctx=None): return rolling_ctx, errors def render_vars(self, in_ctx): - rolling_ctx = json_util.deepcopy(in_ctx) + # only replacing top key values in dict a copy is fine here + rolling_ctx = copy.copy(in_ctx) rendered_vars = {} errors = [] @@ -681,7 +684,7 @@ def render_vars(self, in_ctx): def render_output(self, in_ctx): output_specs = getattr(self, "output") or [] - rolling_ctx = json_util.deepcopy(in_ctx) + rolling_ctx = copy.copy(in_ctx) rendered_outputs = {} errors = [] From d4c848becc95f89b8fc5ca6b03324ea93713ca6e Mon Sep 17 00:00:00 2001 From: guzzijones12 Date: Tue, 30 Jan 2024 20:23:00 +0000 Subject: [PATCH 32/32] flake fix --- orquesta/specs/native/v1/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orquesta/specs/native/v1/models.py b/orquesta/specs/native/v1/models.py index be7f13c4..a77b8155 100644 --- a/orquesta/specs/native/v1/models.py +++ b/orquesta/specs/native/v1/models.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import copy +import logging import six from six.moves import queue