Merge pull request #541 from internetstandards/44

add subdomain suggestions

Makefile

@@ -253,7 +253,7 @@ vulture: ${app} ## Check for unused code
 	${python} -m vulture ${pysrcdirs}
 
 ruff: ${app} ## Faster than black, might autoformat some things
-	${python} -m ruff ${pysrcdirs}
+	${python} -m ruff check ${pysrcdirs}
 
 bandit: ${app} ## Run basic security audit
 	${python} -m bandit --configfile bandit.yaml -r ${pysrcdirs}

dashboard/internet_nl_dashboard/logic/domains.py

@@ -7,6 +7,7 @@
 from typing import Any, Dict, List, Set, Tuple, Union
 
 import pyexcel as p
+import requests
 import tldextract
 from actstream import action
 from celery import group
@@ -28,6 +29,25 @@
 log = logging.getLogger(__package__)
 
 
+def suggest_subdomains(domain: str, period: int = 370):
+    extract = tldextract.extract(domain)
+
+    # ip address or garbage
+    if not extract.domain or not extract.suffix:
+        return []
+
+    # call SUBDOMAIN_SUGGESTION_SERVER_ADDRESS
+    response = requests.get(config.SUBDOMAIN_SUGGESTION_SERVER_ADDRESS,
+                            params={'domain': extract.domain, 'suffix': extract.suffix, 'period': period},
+                            timeout=10)
+
+    if response.status_code != 200:
+        log.error("Failed to retrieve subdomain suggestions from  %s.", config.SUBDOMAIN_SUGGESTION_SERVER_ADDRESS)
+        return []
+
+    return response.json()
+
+
 # todo: write test
 def alter_url_in_urllist(account, data) -> Dict[str, Any]:
     # data = {'list_id': list.id, 'url_id': url.id, 'new_url_string': url.url}

dashboard/internet_nl_dashboard/tests/test_domain_management.py

@@ -6,6 +6,7 @@
 """
 import timeit
 
+import responses
 from constance.test import override_config
 from websecmap.organizations.models import Url
 
@@ -14,10 +15,24 @@
                                                            get_urllist_content, get_urllists_from_account,
                                                            keys_are_present_in_object, rename_list,
                                                            retrieve_possible_urls_from_unfiltered_input,
-                                                           save_urllist_content_by_name)
+                                                           save_urllist_content_by_name, suggest_subdomains)
 from dashboard.internet_nl_dashboard.models import Account
 
 
+@responses.activate
+def test_suggest_subdomains(db, caplog):
+    responses.add(responses.GET, 'http://localhost:8001/?domain=test&suffix=nl&period=370', json=["test"], status=200)
+    responses.add(responses.GET, 'http://localhost:8001/?domain=broken&suffix=nl&period=370', json=[], status=404)
+
+    assert suggest_subdomains("test.nl", 370) == ["test"]
+    assert suggest_subdomains("broken.nl") == []
+    assert "Failed to retrieve" in caplog.text
+
+    # incomplete requests:
+    assert suggest_subdomains("192.168.1.1") == []
+    assert suggest_subdomains("a") == []
+
+
 @override_config(DASHBOARD_MAXIMUM_DOMAINS_PER_LIST=5000)
 def test_add_domains_from_raw_user_data(db, current_path, redis_server):
     """

dashboard/internet_nl_dashboard/urls.py

@@ -54,6 +54,7 @@ def to_url(value):
     path('data/urllist/url/delete/', domains.delete_url_from_urllist_),
     path('data/urllist/download/', domains.download_list_),
     path('data/urllist/upload/<int:list_id>/', spreadsheet.upload_list_),
+    path('data/urllist/suggest-subdomains/', domains.suggest_subdomains_),
 
     path('data/urllist/tag/add/', tags.add_tag_),
     path('data/urllist/tag/remove/', tags.remove_tag_),

dashboard/internet_nl_dashboard/views/app.py

@@ -12,6 +12,17 @@ def config_content():
         "show": {
             "signup_form": configuration["SHOW_SIGNUP_FORM"],
         },
+        "app": {
+            "subdomain_suggestion": {
+                "enabled": configuration["SUBDOMAIN_SUGGESTION_ENABLED"],
+                "default_period": configuration["SUBDOMAIN_SUGGESTION_DEFAULT_TIME_PERIOD"],
+                "default_extend_period": configuration["SUBDOMAIN_SUGGESTION_DEFAULT_EXTEND_TIME_PERIOD"],
+            },
+            # in the future we'll support this
+            "signup": {
+                "enabled": configuration["SHOW_SIGNUP_FORM"],
+            }
+        }
     }
 
 

dashboard/internet_nl_dashboard/views/domains.py

@@ -11,10 +11,18 @@
                                                            delete_url_from_urllist, download_as_spreadsheet,
                                                            get_scan_status_of_list, get_urllist_content,
                                                            get_urllists_from_account, save_urllist_content_by_name,
-                                                           scan_now, update_list_settings)
+                                                           scan_now, suggest_subdomains, update_list_settings)
 from dashboard.internet_nl_dashboard.views import LOGIN_URL, get_account, get_json_body
 
 
+@login_required(login_url=LOGIN_URL)
+def suggest_subdomains_(request) -> JsonResponse:
+    request = get_json_body(request)
+    domain = request.get("domain", "")
+    period = request.get("period", 370)
+    return JsonResponse(suggest_subdomains(domain, period), encoder=JSEncoder, safe=False)
+
+
 @login_required(login_url=LOGIN_URL)
 def get_lists(request) -> JsonResponse:
     return JsonResponse(get_urllists_from_account(account=get_account(request)), encoder=JSEncoder, safe=False)