Merge pull request #41 from cisagov/goose-1.2.1

goose v1.2.1
cisagov · Jun 6, 2023 · 101fc86 · 101fc86
2 parents a1c8485 + 5de03ee
commit 101fc86
Show file tree

Hide file tree

Showing 21 changed files with 757 additions and 193 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,20 @@
 
 All notable changes to this project will be documented in this file.
 
+## [1.2.1] - The goose is loose - 2023-06-06
+### Added
+- Implemented new tables to be pulled from MDE.
+- Added two SBOM files.
+
+### Changed
+- Updated readme with cloud-only account requirement.
+- Better logging for _no_results.json.
+
+### Fixed
+- Fixed Azure government calls.
+- Fixed minor debug logging issues.
+- Fixed the AttributeError encountered during AzureAD calls.
+
 ## [1.2.0] - The goose is loose - 2023-04-21
 ### Added
 - Implemented delegated application authentication.

diff --git a/README.md b/README.md
@@ -79,7 +79,9 @@ python -m venv .venv
 ### Requirements
 The following AzureAD/M365 permissions are required to run Untitled Goose Tool, and provide it read-only access to the tenant.
 
-A user account with the following permissions:
+Please note: The user account should be a cloud-only account (not sync'd to the on-premise environment), this will ensure that the login process stays the same across environments for the tool.
+
+A cloud-only user account with the following permissions:
 
 Exchange Online Admin Center
 ```

diff --git a/cyclonedx_output.json b/cyclonedx_output.json
@@ -0,0 +1,164 @@
+{
+ "$schema": "http://cyclonedx.org/schema/bom-1.4.schema.json",
+ "bomFormat": "CycloneDX",
+ "specVersion": "1.4",
+ "serialNumber": "urn:uuid:e3f22225-e0a8-4ff7-a369-f50caa3e15fb",
+ "version": 1,
+ "metadata": {
+ "timestamp": "2023-06-02T10:38:52-04:00",
+ "tools": [
+ {
+ "vendor": "anchore",
+ "name": "syft",
+ "version": "0.82.0"
+ }
+ ],
+ "component": {
+ "bom-ref": "1964c4ec04b45222",
+ "type": "file",
+ "name": "./untitledgoosetool"
+ }
+ },
+ "components": [
+ {
+ "bom-ref": "pkg:pypi/[email protected]?package-id=36270366523988d",
+ "type": "library",
+ "name": "Gooey",
+ "version": "1.0.8.1",
+ "cpe": "cpe:2.3:a:python-Gooey:python-Gooey:1.0.8.1:*:*:*:*:*:*:*",
+ "purl": "pkg:pypi/[email protected]",
+ "properties": [
+ {
+ "name": "syft:package:foundBy",
+ "value": "python-index-cataloger"
+ },
+ {
+ "name": "syft:package:language",
+ "value": "python"
+ },
+ {
+ "name": "syft:package:type",
+ "value": "python"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:python-Gooey:python_Gooey:1.0.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:python_Gooey:python-Gooey:1.0.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:python_Gooey:python_Gooey:1.0.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:python:python-Gooey:1.0.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:python:python_Gooey:1.0.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:Gooey:python-Gooey:1.0.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:Gooey:python_Gooey:1.0.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:python-Gooey:Gooey:1.0.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:python_Gooey:Gooey:1.0.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:python:Gooey:1.0.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:Gooey:Gooey:1.0.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:location:0:path",
+ "value": "setup.py"
+ }
+ ]
+ },
+ {
+ "bom-ref": "pkg:pypi/[email protected]?package-id=2d83bd819e2e7cb0",
+ "type": "library",
+ "name": "aiohttp",
+ "version": "3.8.1",
+ "cpe": "cpe:2.3:a:python-aiohttp:python-aiohttp:3.8.1:*:*:*:*:*:*:*",
+ "purl": "pkg:pypi/[email protected]",
+ "properties": [
+ {
+ "name": "syft:package:foundBy",
+ "value": "python-index-cataloger"
+ },
+ {
+ "name": "syft:package:language",
+ "value": "python"
+ },
+ {
+ "name": "syft:package:type",
+ "value": "python"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:python-aiohttp:python_aiohttp:3.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:python_aiohttp:python-aiohttp:3.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:python_aiohttp:python_aiohttp:3.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:aiohttp:python-aiohttp:3.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:aiohttp:python_aiohttp:3.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:python-aiohttp:aiohttp:3.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:python_aiohttp:aiohttp:3.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:python:python-aiohttp:3.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:python:python_aiohttp:3.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:aiohttp:aiohttp:3.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:cpe23",
+ "value": "cpe:2.3:a:python:aiohttp:3.8.1:*:*:*:*:*:*:*"
+ },
+ {
+ "name": "syft:location:0:path",
+ "value": "setup.py"
+ }
+ ]
+ }
+ ]
+}
diff --git a/goosey/auth.py b/goosey/auth.py
@@ -19,15 +19,15 @@
 import time
 
 from goosey.utils import *
-from seleniumwire import webdriver
 from selenium.webdriver import FirefoxOptions
 from selenium.webdriver.common.by import By
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from seleniumwire import webdriver
 
 __author__ = "Claire Casalnova, Jordan Eberst, Wellington Lee, Victoria Wallace"
-__version__ = "1.2.0"
+__version__ = "1.2.1"
 
 green = "\x1b[1;32m"
 
@@ -97,9 +97,17 @@ def get_app_resource_uri(self):
  Returns the application resource URI for a commercial or government tenant.
  """
  if self.us_government == 'false':
- return ['https://graph.microsoft.com/.default', 'https://api.securitycenter.microsoft.com/.default', 'https://management.azure.com/.default']
+ if self.mde_gcc == 'false' and self.mde_gcc_high == 'false':
+ return ['https://graph.microsoft.com/.default', 'https://api.securitycenter.microsoft.com/.default', 'https://management.azure.com/.default', 'https://api.security.microsoft.com/.default']
+ elif self.mde_gcc == 'true':
+ return ['https://graph.microsoft.com/.default', 'https://api.securitycenter.microsoft.com/.default', 'https://api-gcc.securitycenter.microsoft.us', 'https://api-gcc.security.microsoft.us']
+ elif self.mde_gcc_high == 'true':
+ return ['https://graph.microsoft.com/.default', 'https://api.securitycenter.microsoft.com/.default', 'https://api-gov.securitycenter.microsoft.us', 'https://api-gov.security.microsoft.us']
  elif self.us_government == 'true':
- return 'https://graph.microsoft.us/.default'
+ if self.mde_gcc == 'true':
+ return ['https://graph.microsoft.us/.default', 'https://management.azure.us/.default', 'https://api-gcc.securitycenter.microsoft.us', 'https://api-gcc.security.microsoft.us']
+ elif self.mde_gcc_high =='true':
+ return ['https://graph.microsoft.us/.default', 'https://management.azure.us/.default', 'https://api-gov.securitycenter.microsoft.us', 'https://api-gov.security.microsoft.us']
 
  def authenticate_device_code_selenium(self):
  """
@@ -532,8 +540,12 @@ def authenticate_mfa_interactive(self):
  self.logger.error("Error obtaining " + cookie_str + ": " + str(e))
 
  for request in browser.requests:
- if request.url == "https://admin.exchange.microsoft.com/beta/UserProfile":
- self.tokendata['validationkey'] = request.headers['validationkey']
+ if self.exo_us_government == 'false':
+ if request.url == "https://admin.exchange.microsoft.com/beta/UserProfile":
+ self.tokendata['validationkey'] = request.headers['validationkey']
+ else:
+ if request.url == "https://admin.exchange.microsoft.us/beta/UserProfile":
+ self.tokendata['validationkey'] = request.headers['validationkey']
 
  if not self.tokendata['validationkey']:
  self.logger.error("Error obtaining validationkey.")
@@ -645,6 +657,8 @@ def parse_config(self, configfile):
  if not self.d4iot:
  self.tenant = config_get(config, 'config', 'tenant', self.logger)
  self.us_government = config_get(config, 'config', 'us_government', self.logger).lower()
+ self.mde_gcc = config_get(config, 'config', 'mde_gcc', self.logger).lower()
+ self.mde_gcc_high = config_get(config, 'config', 'mde_gcc_high', self.logger).lower()
  self.exo_us_government = config_get(config, 'config', 'exo_us_government', self.logger).lower()
  self.subscriptions = config_get(config, 'config', 'subscriptionid', self.logger)
  self.m365 = config_get(config, 'config', 'm365', self.logger).lower()

diff --git a/goosey/azure_ad_datadumper.py b/goosey/azure_ad_datadumper.py
@@ -15,7 +15,7 @@
 from goosey.utils import *
 
 __author__ = "Claire Casalnova, Jordan Eberst, Wellington Lee, Victoria Wallace"
-__version__ = "1.2.0"
+__version__ = "1.2.1"
 
 class AzureAdDataDumper(DataDumper):
 
@@ -144,17 +144,19 @@ async def _dump_signins(self, source: str) -> None:
  f.write(end_time)
  break
 
- except Exception as e:
- self.logger.error('Error on nextLink retrieval: {}'.format(str(e)))
- if e.status:
- if e.status == 429:
- self.logger.info('Sleeping for 60 seconds because of API throttle limit was exceeded.')
- await asyncio.sleep(60)
- retries -= 1
- self.logger.debug('Retries remaining: {}'.format(str(retries)))
- elif e.status == 401:
- self.logger.error('401 unauthorized message received. Exiting calls. Please re-auth.')
- sys.exit(1)
+ except Exception as e: 
+ try:
+ if e.status:
+ if e.status == 429:
+ self.logger.info('Sleeping for 60 seconds because of API throttle limit was exceeded.')
+ await asyncio.sleep(60)
+ retries -= 1
+ self.logger.debug('Retries remaining: {}'.format(str(retries)))
+ elif e.status == 401:
+ self.logger.error('401 unauthorized message received. Exiting calls. Please re-auth.')
+ sys.exit(1)
+ except AttributeError as a:
+ self.logger.error('Error on nextLink retrieval: {}'.format(str(e)))
 
  if os.path.isfile(outfile) and os.stat(outfile).st_size == 0:
  os.remove(outfile) 
@@ -266,15 +268,17 @@ async def dump_azuread_audit(self) -> None:
 
 
  except Exception as e:
- self.logger.error('Error on nextLink retrieval: {}'.format(str(e)))
- if e.status:
- if e.status == 429:
- self.logger.info('Sleeping for 60 seconds because of API throttle limit was exceeded.')
- await asyncio.sleep(60)
- retries -= 1
- elif e.status == 401:
- self.logger.info('401 unauthorized message received. Exiting calls. Please re-auth.')
- sys.exit(1)
+ try:
+ if e.status:
+ if e.status == 429:
+ self.logger.info('Sleeping for 60 seconds because of API throttle limit was exceeded.')
+ await asyncio.sleep(60)
+ retries -= 1
+ elif e.status == 401:
+ self.logger.info('401 unauthorized message received. Exiting calls. Please re-auth.')
+ sys.exit(1)
+ except AttributeError as a:
+ self.logger.error('Error on nextLink retrieval: {}'.format(str(e)))
 
 
  self.logger.info('Finished dumping AzureAD audit logs.')
@@ -452,7 +456,7 @@ async def helper_multiple_object(self, parent, child,identifier='id'):
  f.write(json.dumps(entry, sort_keys=True) + '\n')
  elif not child_list:
  with open(self.failurefile, 'a+', encoding='utf-8') as f:
- f.write(parent + "_" + child + '\n')
+ f.write('No output file: ' + parent + "_" + child + ' - ' + str((datetime.now())) + '\n')
 
  self.logger.info('Finished dumping %s %s information.' % (parent, child))