scheme_check.py
python
sha256:eb10be5b0151404c767863448333d0cf0ad1b63219b5113abb902cb0132ae67e
Merge dev (#242 DOC_INF_04) into task/243-docs-landing (#243)
Opus 5.5
10 days ago
| 1 | """URL-scheme checks for the /muse docs (musehub#242 W2.4, DOC_INF_04). |
| 2 | |
| 3 | A page served over HTTPS must not hand out non-loopback ``http://`` URLs, whether in links, |
| 4 | ``<head>`` metadata, or copy-paste examples, and an API ``url`` field must not either. Loopback |
| 5 | URLs (``http://localhost:1337``, ``http://127.0.0.1``) are legitimate local-development examples. |
| 6 | |
| 7 | Every function is pure: it takes rendered text (or a decoded JSON payload) and returns |
| 8 | human-readable problems, empty when clean. ``test_doc_inf_scheme.py`` feeds it the live pages, |
| 9 | rendered through the production proxy topology, as well as deliberately broken fixtures. |
| 10 | """ |
| 11 | |
| 12 | from __future__ import annotations |
| 13 | |
| 14 | import ipaddress |
| 15 | import re |
| 16 | from collections.abc import Iterator |
| 17 | from html.parser import HTMLParser |
| 18 | from urllib.parse import parse_qsl, urlsplit |
| 19 | |
| 20 | UVICORN_DEFAULT_TRUSTED = "127.0.0.1" # uvicorn's --forwarded-allow-ips default when neither flag nor env sets it |
| 21 | |
| 22 | _URL_RE = re.compile(r"\bhttps?://[^\s\"'<>`]*", re.IGNORECASE) |
| 23 | _TRAILING = ".,;:!?)]}" # prose punctuation that ends a sentence, not a URL |
| 24 | _URL_KEY_RE = re.compile(r"(?:^|_)url$|Url$|URL$|^href$") |
| 25 | |
| 26 | |
| 27 | def is_loopback(host: str) -> bool: |
| 28 | host = host.lower().rstrip(".") |
| 29 | if host == "localhost" or host.endswith(".localhost"): |
| 30 | return True |
| 31 | try: |
| 32 | return ipaddress.ip_address(host).is_loopback |
| 33 | except ValueError: |
| 34 | return False |
| 35 | |
| 36 | |
| 37 | def _problems_for(url: str, nested_in: str | None = None) -> Iterator[str]: |
| 38 | """Problems with one absolute URL, including http URLs percent-encoded in its query string.""" |
| 39 | where = f" (inside the query of {nested_in!r})" if nested_in else "" |
| 40 | try: |
| 41 | parts = urlsplit(url) |
| 42 | host = parts.hostname |
| 43 | except ValueError as exc: |
| 44 | yield f"malformed URL {url!r}{where}: {exc}" |
| 45 | return |
| 46 | if parts.scheme.lower() == "http" and host and not is_loopback(host): |
| 47 | yield f"non-loopback http:// URL {url!r}{where}" |
| 48 | for _, value in parse_qsl(parts.query, keep_blank_values=True): |
| 49 | if _URL_RE.fullmatch(value): |
| 50 | yield from _problems_for(value, nested_in=nested_in or url) |
| 51 | |
| 52 | |
| 53 | def check_text(text: str) -> list[str]: |
| 54 | """Problems with every absolute http(s) URL in ``text``. |
| 55 | |
| 56 | A bare ``http://`` with no host (prose like "you used http:// instead of https://") is a |
| 57 | mention, not a URL, and is not reported. |
| 58 | """ |
| 59 | problems: list[str] = [] |
| 60 | for match in _URL_RE.finditer(text): |
| 61 | problems.extend(_problems_for(match.group(0).rstrip(_TRAILING))) |
| 62 | return problems |
| 63 | |
| 64 | |
| 65 | class _TextParser(HTMLParser): |
| 66 | """Yields ``(location, text)`` for every attribute value and text node, decoded.""" |
| 67 | |
| 68 | def __init__(self) -> None: |
| 69 | super().__init__(convert_charrefs=True) |
| 70 | self.chunks: list[tuple[str, str]] = [] |
| 71 | self._tag = "" |
| 72 | |
| 73 | def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: |
| 74 | self._tag = tag |
| 75 | for name, value in attrs: |
| 76 | if value and name != "xmlns" and not name.startswith("xmlns:"): # namespace names are never fetched |
| 77 | self.chunks.append((f"<{tag} {name}>", value)) |
| 78 | |
| 79 | def handle_data(self, data: str) -> None: |
| 80 | if data.strip(): |
| 81 | self.chunks.append((f"<{self._tag}> text", data)) |
| 82 | |
| 83 | |
| 84 | def check_page(page: str, html: str) -> list[str]: |
| 85 | """Problems in one rendered page, each naming the page, the element/attribute, and the URL.""" |
| 86 | parser = _TextParser() |
| 87 | parser.feed(html) |
| 88 | parser.close() |
| 89 | problems = [f"{page}: {where}: {p}" for where, text in parser.chunks for p in check_text(text)] |
| 90 | return list(dict.fromkeys(problems)) |
| 91 | |
| 92 | |
| 93 | def check_json_urls(source: str, payload: object, _path: str = "$") -> list[str]: |
| 94 | """Problems in every URL-valued field (``url``, ``*_url``, ``*Url``, ``href``) of a decoded JSON payload.""" |
| 95 | problems: list[str] = [] |
| 96 | if isinstance(payload, dict): |
| 97 | for key, value in payload.items(): |
| 98 | here = f"{_path}.{key}" |
| 99 | if isinstance(value, str) and _URL_KEY_RE.search(str(key)): |
| 100 | problems += [f"{source}: {here}: {p}" for p in _problems_for(value)] |
| 101 | else: |
| 102 | problems += check_json_urls(source, value, here) |
| 103 | elif isinstance(payload, list): |
| 104 | for i, item in enumerate(payload): |
| 105 | problems += check_json_urls(source, item, f"{_path}[{i}]") |
| 106 | return problems |
| 107 | |
| 108 | |
| 109 | def production_trusted_proxies(entrypoint: str, dockerfile: str = "") -> list[str]: |
| 110 | """``--forwarded-allow-ips`` in effect for each ``exec uvicorn`` command in the image entrypoint. |
| 111 | |
| 112 | The flag wins; otherwise uvicorn reads ``FORWARDED_ALLOW_IPS`` (here: a Dockerfile ``ENV``); |
| 113 | otherwise it trusts only ``127.0.0.1``. Values set in a host's ``--env-file`` are not visible |
| 114 | to an offline check. |
| 115 | """ |
| 116 | env = re.search(r"^\s*ENV\s+FORWARDED_ALLOW_IPS[=\s]+(['\"]?)([^\s'\"]+)\1", dockerfile, re.MULTILINE) |
| 117 | fallback = env.group(2) if env else UVICORN_DEFAULT_TRUSTED |
| 118 | joined = re.sub(r"\\\n", " ", entrypoint) |
| 119 | values: list[str] = [] |
| 120 | for line in joined.splitlines(): |
| 121 | if not re.match(r"^\s*exec\s+uvicorn\b", line): |
| 122 | continue |
| 123 | flag = re.search(r"--forwarded-allow-ips(?:=|\s+)(['\"]?)([^\s'\"]+)\1", line) |
| 124 | values.append(flag.group(2) if flag else fallback) |
| 125 | return values |
File History
2 commits
sha256:eb10be5b0151404c767863448333d0cf0ad1b63219b5113abb902cb0132ae67e
Merge dev (#242 DOC_INF_04) into task/243-docs-landing (#243)
Opus 5.5
10 days ago
sha256:e8b5a50b0a352faa913ea7d5e72e615527564b52466009c56171d1d54f15b7ec
test(docs): check that /muse pages served over HTTPS hand o…
Opus 5.5
10 days ago