diff --git a/changelogs/fragments/urls-cidr-no-proxy.yml b/changelogs/fragments/urls-cidr-no-proxy.yml new file mode 100644 index 00000000000000..dfe11c8f6b63a4 --- /dev/null +++ b/changelogs/fragments/urls-cidr-no-proxy.yml @@ -0,0 +1,2 @@ +minor_changes: +- urls.py - Support CIDR ranges for no_proxy diff --git a/lib/ansible/module_utils/urls.py b/lib/ansible/module_utils/urls.py index c4c8e3ab7dfaed..1e61cfe57bbb93 100644 --- a/lib/ansible/module_utils/urls.py +++ b/lib/ansible/module_utils/urls.py @@ -37,6 +37,7 @@ import email.policy import email.utils import http.client +import ipaddress import mimetypes import netrc import os @@ -67,6 +68,7 @@ from ansible.module_utils.basic import missing_required_lib from ansible.module_utils.common.collections import Mapping, is_sequence from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text +from ansible.module_utils.common import warnings try: import ssl @@ -300,6 +302,49 @@ def http_open(self, req): return self.do_open(UnixHTTPConnection(self._unix_socket), req) +class ProxyHandler(urllib.request.ProxyHandler): + _SPLITPORT_RE = re.compile('(.*):([0-9]{1,5})', re.DOTALL) + + @classmethod + def _splitport(cls, host): + # Derived from cpython urllib.parse + port = None + if (match := cls._SPLITPORT_RE.fullmatch(host)): + host, port = match.groups() + return host, port or None + + @staticmethod + def _matches(host, port, bypass_network, bypass_port, scheme): + if not port: + port = '443' if scheme == 'https' else '80' + if bypass_port and port != bypass_port: + return False + return host in bypass_network + + def proxy_open(self, req, *args): + """Implements proxy bypassing for cidr ranges""" + hostonly, port = self._splitport(req.host) + try: + req_ip = ipaddress.ip_address(hostonly) + except ValueError: + return super().proxy_open(req, *args) + + no_proxy = self.proxies.get('no') or '' + for bypass in map(str.strip, no_proxy.split(',')): + if '/' not in bypass: + continue + bypass_host, bypass_port = self._splitport(bypass) + try: + bypass_network = ipaddress.ip_network(bypass_host) + except ValueError as e: + warnings.warn(f'no_proxy entry appears to be a CIDR range, but could not be parsed: {e}') + continue + if self._matches(req_ip, port, bypass_network, bypass_port, req.type): + return None + + return super().proxy_open(req, *args) + + class ParseResultDottedDict(dict): ''' A dict that acts similarly to the ParseResult named tuple from urllib @@ -844,9 +889,10 @@ def open(self, method, url, data=None, headers=None, use_proxy=None, headers.update(auth_headers) handlers.extend(auth_handlers) + proxies = None if not use_proxy: - proxyhandler = urllib.request.ProxyHandler({}) - handlers.append(proxyhandler) + proxies = {} + handlers.append(ProxyHandler(proxies)) if not context: context = make_context( diff --git a/test/integration/targets/setup_proxy/files/hamsandwich.py b/test/integration/targets/setup_proxy/files/hamsandwich.py new file mode 100644 index 00000000000000..d3c487b34cefc8 --- /dev/null +++ b/test/integration/targets/setup_proxy/files/hamsandwich.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from proxy.http.proxy import HttpProxyBasePlugin +from proxy.http.parser import HttpParser, httpParserStates + + +class HamSandwichPlugin(HttpProxyBasePlugin): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.parser = None + self.parsable = True + + def handle_upstream_chunk(self, chunk): + if not self.parsable: + return chunk + + if not self.parser: + self.parser = HttpParser.response(chunk) + else: + self.parser.parse(chunk) + + if self.parser.state == httpParserStates.INITIALIZED: + # This is likely TLS without interception + self.parsable = False + return chunk + + if not self.parser.is_complete: + return None + + self.parser.add_header(b'X-Sandwich', b'ham') + return memoryview(bytearray(self.parser.build_response())) diff --git a/test/integration/targets/setup_proxy/handlers/main.yml b/test/integration/targets/setup_proxy/handlers/main.yml new file mode 100644 index 00000000000000..4e1e28e53244c4 --- /dev/null +++ b/test/integration/targets/setup_proxy/handlers/main.yml @@ -0,0 +1,2 @@ +- name: stop proxy.py + command: kill {{ proxy_py_pid }} diff --git a/test/integration/targets/setup_proxy/meta/main.yml b/test/integration/targets/setup_proxy/meta/main.yml new file mode 100644 index 00000000000000..1810d4bec988cb --- /dev/null +++ b/test/integration/targets/setup_proxy/meta/main.yml @@ -0,0 +1,2 @@ +dependencies: + - setup_remote_tmp_dir diff --git a/test/integration/targets/setup_proxy/tasks/main.yml b/test/integration/targets/setup_proxy/tasks/main.yml new file mode 100644 index 00000000000000..7fbba38aaae665 --- /dev/null +++ b/test/integration/targets/setup_proxy/tasks/main.yml @@ -0,0 +1,43 @@ +- name: install proxy.py + pip: + name: proxy.py + virtualenv: '{{ remote_tmp_dir }}/proxy_py' + virtualenv_command: "{{ ansible_python_interpreter }} -m venv" + notify: stop proxy.py + +- name: get venv site-packages + command: >- + {{ remote_tmp_dir }}/proxy_py/bin/python -c 'import site; print(site.getsitepackages()[0])' + register: proxy_py_site_packages + +- name: install proxy.py plugin + copy: + src: hamsandwich.py + dest: '{{ proxy_py_site_packages.stdout }}/hamsandwich.py' + +- name: start proxy.py + command: >- + {{ remote_tmp_dir }}/proxy_py/bin/proxy --port 8080 --log-file "{{ remote_tmp_dir }}/proxy_py/proxy_py.log" + --plugins hamsandwich.HamSandwichPlugin --pid-file "{{ remote_tmp_dir }}/proxy_py/proxy_py.pid" + async: 120 + poll: 0 + register: proxy_py + +- name: wait for proxy.py to start + wait_for: + port: 8080 + connect_timeout: 1 + timeout: 10 + +- name: get proxy.py pid + slurp: + path: '{{ remote_tmp_dir }}/proxy_py/proxy_py.pid' + register: proxy_py_slurp_pid + +- name: set fact for proxy.py pid + set_fact: + proxy_py_pid: '{{ proxy_py_slurp_pid.content|b64decode }}' + +- name: set fact for proxy host + set_fact: + http_proxy: 'http://127.0.0.1:8080' diff --git a/test/integration/targets/uri/aliases b/test/integration/targets/uri/aliases index 90ef161f598809..202ee5fc4a616c 100644 --- a/test/integration/targets/uri/aliases +++ b/test/integration/targets/uri/aliases @@ -1,3 +1,4 @@ destructive shippable/posix/group1 needs/httptester +needs/target/setup_proxy diff --git a/test/integration/targets/uri/tasks/main.yml b/test/integration/targets/uri/tasks/main.yml index a818107d99ca46..1d4f37a0a109cd 100644 --- a/test/integration/targets/uri/tasks/main.yml +++ b/test/integration/targets/uri/tasks/main.yml @@ -727,6 +727,9 @@ - name: Test unix socket import_tasks: install-socat-and-test-unix-socket.yml +- name: Test proxy + import_tasks: proxy.yml + - name: ensure skip action uri: url: http://example.com diff --git a/test/integration/targets/uri/tasks/proxy.yml b/test/integration/targets/uri/tasks/proxy.yml new file mode 100644 index 00000000000000..d92c4582165a40 --- /dev/null +++ b/test/integration/targets/uri/tasks/proxy.yml @@ -0,0 +1,117 @@ +- include_role: + name: setup_proxy + +- name: Get IP address for ansible.http.tests + command: >- + {{ ansible_python_interpreter }} -c 'import socket; print(socket.gethostbyname("{{ httpbin_host }}"))' + register: httpbin_ip + +- name: Test http over http proxy + uri: + url: http://{{ httpbin_host }}/get + environment: + http_proxy: '{{ http_proxy }}' + register: http_over_http + failed_when: http_over_http.x_sandwich is undefined + +- name: Test https over http proxy + uri: + url: https://{{ httpbin_host }}/get + environment: + https_proxy: '{{ http_proxy }}' + register: https_over_http + # failed_when: + # failure checking is handled by the assert at the bottom comparing logs + # because we aren't running a proxy that can inspect the https stream + # there won't be added headers + +- name: Test request without a proxy + uri: + url: http://{{ httpbin_host }}/get + register: request_without_proxy + failed_when: request_without_proxy.x_sandwich is defined + +- name: Test request with proxy and no_proxy=hostname + uri: + url: http://{{ httpbin_host }}/get + environment: + http_proxy: '{{ http_proxy }}' + no_proxy: '{{ httpbin_host }}' + register: no_proxy_hostname + failed_when: no_proxy_hostname.x_sandwich is defined + +- name: Test request with proxy and no_proxy=ip + uri: + url: http://{{ httpbin_ip.stdout }}/get + environment: + http_proxy: '{{ http_proxy }}' + no_proxy: '{{ httpbin_ip.stdout }}' + register: no_proxy_ip + failed_when: no_proxy_ip.x_sandwich is defined + +- name: Test request with proxy and no_proxy=cidr/32 + uri: + url: http://{{ httpbin_ip.stdout }}/get + environment: + http_proxy: '{{ http_proxy }}' + no_proxy: '{{ httpbin_ip.stdout }}/32' + register: no_proxy_cidr_32 + failed_when: no_proxy_cidr_32.x_sandwich is defined + +- name: Test request with proxy and no_proxy=cidr/24 + uri: + url: http://{{ httpbin_ip.stdout }}/get + environment: + http_proxy: '{{ http_proxy }}' + no_proxy: '{{ httpbin_cidr }}' + register: no_proxy_cidr_24 + vars: + httpbin_cidr: "{{ httpbin_ip.stdout.split('.')[:3]|join('.') }}.0/24" + failed_when: no_proxy_cidr_24.x_sandwich is defined + +- name: Test request with proxy and non-matching no_proxy=cidr + uri: + url: http://{{ httpbin_ip.stdout }}/get + environment: + http_proxy: '{{ http_proxy }}' + no_proxy: 1.2.3.0/24 + register: no_proxy_non_matching_cidr + failed_when: no_proxy_non_matching_cidr.x_sandwich is undefined + +- name: Test request with proxy and no_proxy=cidr:port + uri: + url: http://{{ httpbin_ip.stdout }}/get + environment: + http_proxy: '{{ http_proxy }}' + no_proxy: '{{ httpbin_ip.stdout }}/32:80' + register: no_proxy_cidr_port + failed_when: no_proxy_cidr_port.x_sandwich is defined + +- name: Test request with proxy and non-matching no_proxy=cidr:port + uri: + url: http://{{ httpbin_ip.stdout }}/get + environment: + http_proxy: '{{ http_proxy }}' + no_proxy: '{{ httpbin_ip.stdout }}/32:8080' + register: no_proxy_non_matching_cidr_port + failed_when: no_proxy_non_matching_cidr_port.x_sandwich is undefined + +- slurp: + path: "{{ remote_tmp_dir }}/proxy_py/proxy_py.log" + register: proxy_py_logs + +- debug: + msg: '{{ proxy_py_logs.content|b64decode }}' + +- assert: + that: + - >- + log_content is contains "CONNECT " ~ httpbin_host ~ ":443" + # https over http + - >- + log_content|regex_findall("CONNECT ")|length == 1 + # 3 http over http + - >- + log_content|regex_findall('GET')|length == 3 + vars: + log_content: '{{ proxy_py_logs.content|b64decode }}'