StackStorm Packs - Support for Git Submodules

CHANGELOG.rst

@@ -28,6 +28,8 @@ Added
 * Expose environment variable ST2_ACTION_DEBUG to all StackStorm actions.
  Contributed by @maxfactor1
 
+* Added option to checkout git submodules when downloading/installing packs #5814
+ Contributed by @jk464
 
 3.8.0 - November 18, 2022
 -------------------------

contrib/packs/actions/download.yaml

@@ -28,3 +28,8 @@
  items:
  type: "string"
  required: false
+ checkout_submodules:
+ type: "boolean"
+ description: "Set to True to checkout git submodules present in the pack"
+ required: false
+ default: false

contrib/packs/actions/install.meta.yaml

@@ -37,4 +37,9 @@
  required: false
  description: Action timeout in seconds. Action will get killed if it doesn't finish in timeout
  type: integer
+ checkout_submodules:
+ type: "boolean"
+ description: "Set to True to checkout git submodules present in the pack"
+ required: false
+ default: false
 

contrib/packs/actions/pack_mgmt/download.py

@@ -71,7 +71,13 @@ def __init__(self, config=None, action_service=None):
  os.environ["no_proxy"] = self.no_proxy
 
  def run(
- self, packs, abs_repo_base, verifyssl=True, force=False, dependency_list=None
+ self,
+ packs,
+ abs_repo_base,
+ verifyssl=True,
+ force=False,
+ dependency_list=None,
+ checkout_submodules=False,
  ):
  result = {}
  pack_url = None
@@ -86,6 +92,7 @@ def run(
  proxy_config=self.proxy_config,
  force_permissions=True,
  logger=self.logger,
+ checkout_submodules=checkout_submodules,
  )
  pack_url, pack_ref, pack_result = pack_result
  result[pack_ref] = pack_result
@@ -99,6 +106,7 @@ def run(
  proxy_config=self.proxy_config,
  force_permissions=True,
  logger=self.logger,
+ checkout_submodules=checkout_submodules,
  )
  pack_url, pack_ref, pack_result = pack_result
  result[pack_ref] = pack_result

contrib/packs/actions/workflows/install.yaml

@@ -9,6 +9,7 @@ input:
  - force
  - skip_dependencies
  - timeout
+ - checkout_submodules
 
 vars:
  - packs_list: null
@@ -30,6 +31,7 @@ tasks:
  packs: <% ctx().packs %>
  force: <% ctx().force %>
  dependency_list: <% ctx().dependency_list %>
+ checkout_submodules: <% ctx().checkout_submodules %>
  next:
  - when: <% succeeded() %>
  do: make_a_prerun

st2api/st2api/controllers/v1/packs.py

@@ -103,6 +103,9 @@ def post(self, pack_install_request, requester_user=None):
  if pack_install_request.skip_dependencies:
  parameters["skip_dependencies"] = True
 
+ if pack_install_request.checkout_submodules:
+ parameters["checkout_submodules"] = True
+
  if not requester_user:
  requester_user = UserDB(name=cfg.CONF.system_user.user)
 

st2api/tests/unit/controllers/v1/test_packs.py

@@ -210,6 +210,18 @@ def test_install_with_skip_dependencies_parameter(self, _handle_schedule_executi
  self.assertEqual(resp.status_int, 202)
  self.assertEqual(resp.json, {"execution_id": "123"})
 
+ @mock.patch.object(ActionExecutionsControllerMixin, "_handle_schedule_execution")
+ def test_install_with_checkout_submodules_parameter(
+ self, _handle_schedule_execution
+ ):
+ _handle_schedule_execution.return_value = Response(json={"id": "123"})
+ payload = {"packs": ["some"], "checkout_submodules": True}
+
+ resp = self.app.post_json("/v1/packs/install", payload)
+
+ self.assertEqual(resp.status_int, 202)
+ self.assertEqual(resp.json, {"execution_id": "123"})
+
  @mock.patch.object(ActionExecutionsControllerMixin, "_handle_schedule_execution")
  def test_uninstall(self, _handle_schedule_execution):
  _handle_schedule_execution.return_value = Response(json={"id": "123"})

st2client/st2client/commands/pack.py

@@ -266,6 +266,12 @@ def __init__(self, resource, *args, **kwargs):
  default=False,
  help="Skip pack dependency installation.",
  )
+ self.parser.add_argument(
+ "--checkout-submodules",
+ action="store_true",
+ default=False,
+ help="Checkout git submodules present in the pack.",
+ )
 
  def run(self, args, **kwargs):
  is_structured_output = args.json or args.yaml
@@ -279,6 +285,7 @@ def run(self, args, **kwargs):
  args.packs,
  force=args.force,
  skip_dependencies=args.skip_dependencies,
+ checkout_submodules=args.checkout_submodules,
  **kwargs,
  )
 

st2client/st2client/models/core.py

@@ -628,12 +628,20 @@ class AsyncRequest(Resource):
 
 class PackResourceManager(ResourceManager):
  @add_auth_token_to_kwargs_from_env
- def install(self, packs, force=False, skip_dependencies=False, **kwargs):
+ def install(
+ self,
+ packs,
+ force=False,
+ skip_dependencies=False,
+ checkout_submodules=False,
+ **kwargs,
+ ):
  url = "/%s/install" % (self.resource.get_url_path_name())
  payload = {
  "packs": packs,
  "force": force,
  "skip_dependencies": skip_dependencies,
+ "checkout_submodules": checkout_submodules,
  }
  response = self.client.post(url, payload, **kwargs)
  if response.status_code != http_client.OK:

st2common/st2common/models/api/pack.py

@@ -394,6 +394,11 @@ class PackInstallRequestAPI(BaseAPI):
  "description": "Set to True to skip pack dependency installations.",
  "default": False,
  },
+ "checkout_submodules": {
+ "type": "boolean",
+ "description": "Checkout git submodules present in the pack.",
+ "default": False,
+ },
  },
  }
 

st2common/st2common/openapi.yaml

@@ -5173,6 +5173,10 @@ definitions:
  type: boolean
  description: Set to True to skip pack dependency installations.
  default: false
+ checkout_submodules:
+ type: boolean
+ description: Set to True to checkout git submodules present in the pack.
+ default: false
  required:
  - packs
  PacksUninstall:

st2common/st2common/openapi.yaml.j2

@@ -5169,6 +5169,10 @@ definitions:
  type: boolean
  description: Set to True to skip pack dependency installations.
  default: false
+ checkout_submodules:
+ type: boolean
+ description: Set to True to checkout git submodules present in the pack.
+ default: false
  required:
  - packs
  PacksUninstall:

st2common/st2common/util/pack_management.py

@@ -79,6 +79,7 @@ def download_pack(
  force_owner_group=True,
  force_permissions=True,
  logger=LOG,
+ checkout_submodules=False,
 ):
  """
  Download the pack and move it to /opt/stackstorm/packs.
@@ -99,6 +100,10 @@ def download_pack(
  :param force: Force the installation and ignore / delete the lock file if it already exists.
  :type force: ``bool``
 
+ :param checkout_submodules: Whether to also checkout git submodules present
+ in the pack
+ :type checkout_submodules: ``bool``
+
  :return: (pack_url, pack_ref, result)
  :rtype: tuple
  """
@@ -163,6 +168,7 @@ def download_pack(
  repo_url=pack_url,
  verify_ssl=verify_ssl,
  ref=pack_version,
+ checkout_submodules=checkout_submodules,
  )
 
  pack_metadata = get_pack_metadata(pack_dir=abs_local_path)
@@ -190,7 +196,9 @@ def download_pack(
  return tuple(result)
 
 
-def clone_repo(temp_dir, repo_url, verify_ssl=True, ref="master"):
+def clone_repo(
+ temp_dir, repo_url, verify_ssl=True, ref="master", checkout_submodules=False
+):
  # Switch to non-interactive mode
  os.environ["GIT_TERMINAL_PROMPT"] = "0"
  os.environ["GIT_ASKPASS"] = "/bin/echo"
@@ -204,6 +212,10 @@ def clone_repo(temp_dir, repo_url, verify_ssl=True, ref="master"):
  # future.
  repo = Repo.clone_from(repo_url, temp_dir)
 
+ # Checkout any Git Submodules if requested.
+ if checkout_submodules:
+ repo.submodule_update(recursive=False)
+
  is_local_repo = repo_url.startswith("file://")
 
  try: