Merge pull request #528 from internetstandards/44

44

requirements-deploy.in

@@ -3,10 +3,17 @@
 # make sure there are no version conflicts with requirements.txt
 -c requirements.txt
 
-websecmap[deploy] @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@5a97fb289d0e86b444d98cf6d30fbef3b964120b
+websecmap[deploy] @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@c61e40b5681bf5e09be8f3373dc7ea7072d59e2f
 
 # include (security) version constraints for non primary dependencies
 -c security-constraints.in
 
 uwsgi
 django-uwsgi
+
+
+# Scanner dep needed to parse SNI: when performing raw host requests the cert hostname also needs to be correct
+# see: https://toolbelt.readthedocs.io/en/latest/adapters.html#hostheaderssladapter
+# using unreleased version from github which has better support for SNI server_hostname which prevents connection
+# resets on some servers
+requests_toolbelt @ git+https://github.com/requests/toolbelt@c73ad2c204a2e0cd5617a836ac536d5e06778ab0

requirements-deploy.txt

@@ -524,7 +524,7 @@ requests-file==1.5.1
     #   -c requirements.txt
     #   tldextract
     #   websecmap
-requests-toolbelt==1.0.0
+requests-toolbelt @ git+https://github.com/requests/toolbelt@c73ad2c204a2e0cd5617a836ac536d5e06778ab0
     # via
     #   -c requirements.txt
     #   websecmap
@@ -641,7 +641,7 @@ wcwidth==0.2.13
     #   -c requirements.txt
     #   prompt-toolkit
     #   websecmap
-websecmap[deploy] @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@5a97fb289d0e86b444d98cf6d30fbef3b964120b
+websecmap[deploy] @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@c61e40b5681bf5e09be8f3373dc7ea7072d59e2f
     # via
     #   -c requirements.txt
     #   -r requirements-deploy.in

requirements-dev.in

@@ -18,7 +18,7 @@ django-extensions
 django-debug-toolbar
 django-debug-toolbar-request-history
 pytest-mock
-websecmap @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@5a97fb289d0e86b444d98cf6d30fbef3b964120b
+websecmap @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@c61e40b5681bf5e09be8f3373dc7ea7072d59e2f
 # use the version with relaxed attr requirements
 pytest-docker @ git+https://github.com/avast/pytest-docker@519b155009b6f3570c01f1f56e7c4e6ce3c5c760
 
@@ -59,4 +59,11 @@ types-setuptools
 packaging>=23.1
 
 # include (security) version constraints for non primary dependencies
--c security-constraints.in
+-c security-constraints.in
+
+
+# Scanner dep needed to parse SNI: when performing raw host requests the cert hostname also needs to be correct
+# see: https://toolbelt.readthedocs.io/en/latest/adapters.html#hostheaderssladapter
+# using unreleased version from github which has better support for SNI server_hostname which prevents connection
+# resets on some servers
+requests_toolbelt @ git+https://github.com/requests/toolbelt@c73ad2c204a2e0cd5617a836ac536d5e06778ab0

requirements-dev.txt

@@ -110,6 +110,7 @@ chardet==5.1.0
 charset-normalizer==3.1.0
     # via
     #   -c requirements.txt
+    #   niquests
     #   requests
     #   websecmap
 click==8.1.7
@@ -357,7 +358,11 @@ greenlet==3.0.3
     #   gevent
     #   websecmap
 h11==0.14.0
-    # via uvicorn
+    # via
+    #   -c requirements.txt
+    #   urllib3-future
+    #   uvicorn
+    #   websecmap
 httmock==1.4.0
     # via -r requirements-dev.in
 humanize==4.9.0
@@ -369,6 +374,7 @@ idna==3.4
     # via
     #   -c requirements.txt
     #   anyio
+    #   niquests
     #   requests
     #   tldextract
     #   websecmap
@@ -387,6 +393,11 @@ isort==5.13.2
     # via
     #   -r requirements-dev.in
     #   pylint
+jh2==5.0.3
+    # via
+    #   -c requirements.txt
+    #   urllib3-future
+    #   websecmap
 jinja2==3.1.4
     # via
     #   rst2pdf
@@ -395,6 +406,11 @@ jsonfield==3.1.0
     # via
     #   -c requirements.txt
     #   websecmap
+kiss-headers==2.4.3
+    # via
+    #   -c requirements.txt
+    #   niquests
+    #   websecmap
 kombu==5.3.7
     # via
     #   -c requirements.txt
@@ -455,6 +471,10 @@ netaddr==0.8.0
     #   -c requirements.txt
     #   dnsrecon
     #   websecmap
+niquests==3.6.6
+    # via
+    #   -c requirements.txt
+    #   websecmap
 numpy==1.25.0
     # via
     #   -c requirements.txt
@@ -661,6 +681,11 @@ pyyaml==6.0
     #   rst2pdf
     #   tablib
     #   websecmap
+qh3==1.0.7
+    # via
+    #   -c requirements.txt
+    #   urllib3-future
+    #   websecmap
 rdp==0.8
     # via
     #   -c requirements.txt
@@ -694,9 +719,10 @@ requests-file==1.5.1
     #   -c requirements.txt
     #   tldextract
     #   websecmap
-requests-toolbelt==1.0.0
+requests-toolbelt @ git+https://github.com/requests/toolbelt@c73ad2c204a2e0cd5617a836ac536d5e06778ab0
     # via
     #   -c requirements.txt
+    #   -r requirements-dev.in
     #   websecmap
 responses==0.25.0
     # via pytest-responses
@@ -876,6 +902,11 @@ urllib3==1.26.18
     #   responses
     #   sentry-sdk
     #   websecmap
+urllib3-future==2.7.912
+    # via
+    #   -c requirements.txt
+    #   niquests
+    #   websecmap
 uvicorn==0.29.0
     # via sphinx-autobuild
 validators==0.20.0
@@ -891,14 +922,19 @@ vine==5.1.0
     #   websecmap
 vulture==2.11
     # via -r requirements-dev.in
+wassima==1.1.1
+    # via
+    #   -c requirements.txt
+    #   niquests
+    #   websecmap
 watchfiles==0.21.0
     # via sphinx-autobuild
 wcwidth==0.2.13
     # via
     #   -c requirements.txt
     #   prompt-toolkit
     #   websecmap
-websecmap @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@5a97fb289d0e86b444d98cf6d30fbef3b964120b
+websecmap @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@c61e40b5681bf5e09be8f3373dc7ea7072d59e2f
     # via
     #   -c requirements.txt
     #   -r requirements-dev.in

requirements.in

@@ -9,7 +9,7 @@
 # - no version pinning, unless it is required and explained
 
 # to update websecmap, update the SHA hash and run: make update_requirement_websecmap
-websecmap @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@5a97fb289d0e86b444d98cf6d30fbef3b964120b
+websecmap @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@c61e40b5681bf5e09be8f3373dc7ea7072d59e2f
 
 django-otp
 django-two-factor-auth>1.15
@@ -53,3 +53,9 @@ openpyxl==3.0.10
 
 # password change forms, saml oauth and more
 django-allauth
+
+# Scanner dep needed to parse SNI: when performing raw host requests the cert hostname also needs to be correct
+# see: https://toolbelt.readthedocs.io/en/latest/adapters.html#hostheaderssladapter
+# using unreleased version from github which has better support for SNI server_hostname which prevents connection
+# resets on some servers
+requests_toolbelt @ git+https://github.com/requests/toolbelt@c73ad2c204a2e0cd5617a836ac536d5e06778ab0

requirements.txt

@@ -440,8 +440,10 @@ requests-file==1.5.1
     # via
     #   tldextract
     #   websecmap
-requests-toolbelt==1.0.0
-    # via websecmap
+requests-toolbelt @ git+https://github.com/requests/toolbelt@c73ad2c204a2e0cd5617a836ac536d5e06778ab0
+    # via
+    #   -r requirements.in
+    #   websecmap
 retry==0.9.2
     # via websecmap
 sentry-sdk==1.9.0
@@ -533,7 +535,7 @@ wcwidth==0.2.13
     # via
     #   prompt-toolkit
     #   websecmap
-websecmap @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@5a97fb289d0e86b444d98cf6d30fbef3b964120b
+websecmap @ git+https://gitlab.com/internet-cleanup-foundation/web-security-map@c61e40b5681bf5e09be8f3373dc7ea7072d59e2f
     # via -r requirements.in
 wikidata==0.7.0
     # via websecmap