Add bmchelix source

cartography/cli.py

@@ -220,6 +220,30 @@ def _build_parser(self):
                 ' If not specified, cartography by default will run all AWS sync modules available.'
             ),
         )
+        parser.add_argument(
+            '--bmchelix-token-env-var',
+            type=str,
+            default=None,
+            help=(
+                'The name of environment variable containing the bmc helix bearer token for authentication.'
+            ),
+        )
+        parser.add_argument(
+            '--bmchelix-api-url',
+            type=str,
+            default=None,
+            help=(
+                'The url of the target BMC Helix instance usually https://INSTANCE.onbmc.com/api/v1.8'
+            ),
+        )
+        parser.add_argument(
+            '--bmchelix-verify-cert',
+            type=bool,
+            default=True,
+            help=(
+                'Validate https certificate of BMC Helix server.'
+            ),
+        )
         parser.add_argument(
             '--crxcavator-api-base-uri',
             type=str,
@@ -597,6 +621,15 @@ def main(self, argv: str) -> int:
         else:
             config.azure_client_secret = None
 
+        # BMC Helix config
+        if config.bmchelix_token_env_var:
+            logger.debug(
+                f"Reading token for BMC Helix from environment variable {config.bmchelix_token_env_var}",
+            )
+            config.bmchelix_token = os.environ.get(config.bmchelix_token_env_var)
+        else:
+            config.bmchelix_token = None
+
         # Okta config
         if config.okta_org_id and config.okta_api_key_env_var:
             logger.debug(f"Reading API key for Okta from environment variable {config.okta_api_key_env_var}")

cartography/config.py

@@ -131,6 +131,9 @@ def __init__(
         azure_client_secret=None,
         aws_requested_syncs=None,
         analysis_job_directory=None,
+        bmchelix_token=None,
+        bmchelix_api_url=None,
+        bmchelix_verify_cert=None,
         crxcavator_api_base_uri=None,
         crxcavator_api_key=None,
         oci_sync_all_profiles=None,
@@ -187,6 +190,9 @@ def __init__(
         self.azure_client_secret = azure_client_secret
         self.aws_requested_syncs = aws_requested_syncs
         self.analysis_job_directory = analysis_job_directory
+        self.bmchelix_token = bmchelix_token
+        self.bmchelix_api_url = bmchelix_api_url
+        self.bmchelix_verify_cert = bmchelix_verify_cert
         self.crxcavator_api_base_uri = crxcavator_api_base_uri
         self.crxcavator_api_key = crxcavator_api_key
         self.oci_sync_all_profiles = oci_sync_all_profiles

cartography/data/indexes.cypher

@@ -410,3 +410,5 @@ CREATE INDEX IF NOT EXISTS FOR (n:KubernetesSecret) ON (n.lastupdated);
 CREATE INDEX IF NOT EXISTS FOR (n:KubernetesService) ON (n.id);
 CREATE INDEX IF NOT EXISTS FOR (n:KubernetesService) ON (n.name);
 CREATE INDEX IF NOT EXISTS FOR (n:KubernetesService) ON (n.lastupdated);
+CREATE INDEX IF NOT EXISTS FOR (n:BmcHelixHost) ON (n.uuid);
+CREATE INDEX IF NOT EXISTS FOR (n:BmcHelixHost) ON (n.lastupdated);

cartography/data/jobs/cleanup/bmchelix_import_cleanup.json

@@ -0,0 +1,10 @@
+{
+  "statements": [
+    {
+      "query": "WITH datetime()-duration('P7D') AS threshold MATCH (h:BmcHelixHost) WHERE h.lastupdated < threshold WITH h LIMIT $LIMIT_SIZE DETACH DELETE (h)",
+      "iterative": true,
+      "iterationsize": 100
+    }
+  ],
+  "name": "cleanup bmchelix"
+}

cartography/intel/bmchelix/__init__.py

@@ -0,0 +1,63 @@
+"""
+cartography/intel/bmchelix
+"""
+import logging
+
+import neo4j
+
+from cartography.config import Config
+from cartography.intel.bmchelix.endpoints import sync_hosts
+from cartography.stats import get_stats_client
+from cartography.util import merge_module_sync_metadata
+from cartography.util import run_cleanup_job
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+stat_handler = get_stats_client(__name__)
+
+
+@timeit
+def start_bmchelix_ingestion(
+    neo4j_session: neo4j.Session,
+    config: Config,
+) -> None:
+    """
+    Perform ingestion of bmchelix data.
+    :param neo4j_session: Neo4J session for database interface
+    :param config: A cartography.config object
+    :return: None
+    """
+    common_job_parameters = {
+        "UPDATE_TAG": config.update_tag,
+    }
+    if not config.bmchelix_token:
+        logger.error("bmchelix config not found")
+        return
+
+    authorization = (
+        config.bmchelix_token,
+        config.bmchelix_api_url,
+        config.bmchelix_verify_cert,
+    )
+    sync_hosts(
+        neo4j_session,
+        config.update_tag,
+        authorization,
+    )
+    run_cleanup_job(
+        "bmchelix_import_cleanup.json",
+        neo4j_session,
+        common_job_parameters,
+    )
+
+    group_id = "public"
+    if config.bmchelix_api_url:
+        group_id = config.bmchelix_api_url
+    merge_module_sync_metadata(
+        neo4j_session,
+        group_type="bmchelix",
+        group_id=group_id,
+        synced_type="bmchelix",
+        update_tag=config.update_tag,
+        stat_handler=stat_handler,
+    )

cartography/intel/bmchelix/endpoints.py

@@ -0,0 +1,140 @@
+"""
+cartography/intel/bmchelix/endpoints
+"""
+# pylint: disable=missing-function-docstring,too-many-arguments
+import logging
+from typing import Dict
+from typing import List
+from typing import Tuple
+
+import neo4j
+
+from .util import get_bmchelix_hosts
+from .util import get_bmchelix_virtualmachines
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+def sync_hosts(
+    neo4j_session: neo4j.Session,
+    update_tag: int,
+    authorization: Tuple[str, str, bool],
+) -> None:
+    bmchelix_hosts_list = get_bmchelix_hosts(authorization)
+    for host_data in bmchelix_hosts_list:
+        load_host_data(neo4j_session, host_data, update_tag)
+
+    bmchelix_vm_list = get_bmchelix_virtualmachines(authorization)
+    for host_data in bmchelix_vm_list:
+        load_vm_data(neo4j_session, host_data, update_tag)
+
+
+def load_host_data(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
+) -> None:
+    """
+    Transform and load scan information
+
+    FIXME!
+    neo4j.exceptions.CypherTypeError: {code: Neo.ClientError.Statement.TypeError}
+    {message: Collections containing null values can not be stored in properties.}
+    = how to identify problematic row/column?
+    """
+    ingestion_cypher_query = """
+    UNWIND $Hosts AS host
+        MERGE (h:BmcHelixHost{bmchelix_uuid: host.uuid})
+        ON CREATE SET h.bmchelix_uuid = host.uuid,
+            h.firstseen = timestamp()
+        SET h.status = host.status,
+            h.hostname = host.name,
+            h.short_hostname = host.short_hostname,
+            h.platform_name = host.vm_os_type,
+            h.os = host.vm_os,
+            h.hw_vendor = host.hw_vendor,
+            h.virtual = host.virtual,
+            h.cloud = host.cloud,
+            h.tool_first_seen = host.firstSeen,
+            h.tool_last_seen = host.tool_last_seen,
+            h.vm_power_state = host.vm_power_state,
+            h.instance_id = host.instance_id,
+            h.subscription_id = host.subscription_id,
+            h.resource_group = host.resource_group,
+            h.resource_id = host.vmMetadata_resourceId,
+            h.tags_costcenter = host.tags_costcenter,
+            h.tags_engcontact = host.tags_engcontact,
+            h.tags_businesscontact = host.tags_businesscontact,
+            h.modified_timestamp = host.modified_timestamp,
+            h.lastupdated = $update_tag
+        WITH h
+        MATCH (s:AzureSubscription{id: h.subscription_id})
+        MERGE (s)-[r:CONTAINS]->(h)
+        ON CREATE SET r.firstseen = timestamp()
+        SET r.lastupdated = $update_tag
+        WITH h
+        MATCH (s:AzureVirtualMachine{id: h.resource_id})
+        MERGE (s)-[r:PRESENT_IN]->(h)
+        ON CREATE SET r.firstseen = timestamp()
+        SET r.lastupdated = $update_tag
+    """
+    logger.debug("Loading %s bmchelix hosts.", len(data))
+    neo4j_session.run(
+        ingestion_cypher_query,
+        Hosts=data,
+        update_tag=update_tag,
+    )
+
+
+def load_vm_data(
+    neo4j_session: neo4j.Session,
+    data: List[Dict],
+    update_tag: int,
+) -> None:
+    """
+    Transform and load scan information
+    """
+    ingestion_cypher_query = """
+    UNWIND $Hosts AS host
+        MERGE (h:BmcHelixVirtualMachines{bmchelix_uuid: host.uuid})
+        ON CREATE SET h.bmchelix_uuid = host.uuid,
+            h.firstseen = timestamp()
+        SET h.status = host.status,
+            h.hostname = host.hostname,
+            h.short_hostname = host.short_hostname,
+            h.platform_name = host.vm_os_type,
+            h.os = host.vm_os,
+            h.hw_vendor = host.hw_vendor,
+            h.virtual = host.virtual,
+            h.cloud = host.cloud,
+            h.tool_first_seen = host.firstSeen,
+            h.tool_last_seen = host.tool_last_seen,
+            h.vm_power_state = host.vm_power_state,
+            h.instance_id = host.instance_id,
+            h.subscription_id = host.subscription_id,
+            h.resource_group = host.resource_group,
+            h.resource_id = host.resource_id,
+            h.tags_costcenter = host.tags_costcenter,
+            h.tags_engcontact = host.tags_engcontact,
+            h.tags_businesscontact = host.tags_businesscontact,
+            h.modified_timestamp = host.modified_timestamp,
+            h.lastupdated = $update_tag
+        WITH h
+        MATCH (s:AzureSubscription{id: h.subscription_id})
+        MERGE (s)-[r:CONTAINS]->(h)
+        ON CREATE SET r.firstseen = timestamp()
+        SET r.lastupdated = $update_tag
+        WITH h
+        MATCH (s:AzureVirtualMachine{id: h.resource_id})
+        MERGE (s)-[r:PRESENT_IN]->(h)
+        ON CREATE SET r.firstseen = timestamp()
+        SET r.lastupdated = $update_tag
+    """
+    logger.debug("Loading %s bmchelix virtual machines.", len(data))
+    neo4j_session.run(
+        ingestion_cypher_query,
+        Hosts=data,
+        update_tag=update_tag,
+    )