gabriel / musehub public
scheme_check.py python
125 lines 5.2 KB
Raw
sha256:eb10be5b0151404c767863448333d0cf0ad1b63219b5113abb902cb0132ae67e Merge dev (#242 DOC_INF_04) into task/243-docs-landing (#243) Opus 5.5 10 days ago
1 """URL-scheme checks for the /muse docs (musehub#242 W2.4, DOC_INF_04).
2
3 A page served over HTTPS must not hand out non-loopback ``http://`` URLs, whether in links,
4 ``<head>`` metadata, or copy-paste examples, and an API ``url`` field must not either. Loopback
5 URLs (``http://localhost:1337``, ``http://127.0.0.1``) are legitimate local-development examples.
6
7 Every function is pure: it takes rendered text (or a decoded JSON payload) and returns
8 human-readable problems, empty when clean. ``test_doc_inf_scheme.py`` feeds it the live pages,
9 rendered through the production proxy topology, as well as deliberately broken fixtures.
10 """
11
12 from __future__ import annotations
13
14 import ipaddress
15 import re
16 from collections.abc import Iterator
17 from html.parser import HTMLParser
18 from urllib.parse import parse_qsl, urlsplit
19
20 UVICORN_DEFAULT_TRUSTED = "127.0.0.1" # uvicorn's --forwarded-allow-ips default when neither flag nor env sets it
21
22 _URL_RE = re.compile(r"\bhttps?://[^\s\"'<>`]*", re.IGNORECASE)
23 _TRAILING = ".,;:!?)]}" # prose punctuation that ends a sentence, not a URL
24 _URL_KEY_RE = re.compile(r"(?:^|_)url$|Url$|URL$|^href$")
25
26
27 def is_loopback(host: str) -> bool:
28 host = host.lower().rstrip(".")
29 if host == "localhost" or host.endswith(".localhost"):
30 return True
31 try:
32 return ipaddress.ip_address(host).is_loopback
33 except ValueError:
34 return False
35
36
37 def _problems_for(url: str, nested_in: str | None = None) -> Iterator[str]:
38 """Problems with one absolute URL, including http URLs percent-encoded in its query string."""
39 where = f" (inside the query of {nested_in!r})" if nested_in else ""
40 try:
41 parts = urlsplit(url)
42 host = parts.hostname
43 except ValueError as exc:
44 yield f"malformed URL {url!r}{where}: {exc}"
45 return
46 if parts.scheme.lower() == "http" and host and not is_loopback(host):
47 yield f"non-loopback http:// URL {url!r}{where}"
48 for _, value in parse_qsl(parts.query, keep_blank_values=True):
49 if _URL_RE.fullmatch(value):
50 yield from _problems_for(value, nested_in=nested_in or url)
51
52
53 def check_text(text: str) -> list[str]:
54 """Problems with every absolute http(s) URL in ``text``.
55
56 A bare ``http://`` with no host (prose like "you used http:// instead of https://") is a
57 mention, not a URL, and is not reported.
58 """
59 problems: list[str] = []
60 for match in _URL_RE.finditer(text):
61 problems.extend(_problems_for(match.group(0).rstrip(_TRAILING)))
62 return problems
63
64
65 class _TextParser(HTMLParser):
66 """Yields ``(location, text)`` for every attribute value and text node, decoded."""
67
68 def __init__(self) -> None:
69 super().__init__(convert_charrefs=True)
70 self.chunks: list[tuple[str, str]] = []
71 self._tag = ""
72
73 def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
74 self._tag = tag
75 for name, value in attrs:
76 if value and name != "xmlns" and not name.startswith("xmlns:"): # namespace names are never fetched
77 self.chunks.append((f"<{tag} {name}>", value))
78
79 def handle_data(self, data: str) -> None:
80 if data.strip():
81 self.chunks.append((f"<{self._tag}> text", data))
82
83
84 def check_page(page: str, html: str) -> list[str]:
85 """Problems in one rendered page, each naming the page, the element/attribute, and the URL."""
86 parser = _TextParser()
87 parser.feed(html)
88 parser.close()
89 problems = [f"{page}: {where}: {p}" for where, text in parser.chunks for p in check_text(text)]
90 return list(dict.fromkeys(problems))
91
92
93 def check_json_urls(source: str, payload: object, _path: str = "$") -> list[str]:
94 """Problems in every URL-valued field (``url``, ``*_url``, ``*Url``, ``href``) of a decoded JSON payload."""
95 problems: list[str] = []
96 if isinstance(payload, dict):
97 for key, value in payload.items():
98 here = f"{_path}.{key}"
99 if isinstance(value, str) and _URL_KEY_RE.search(str(key)):
100 problems += [f"{source}: {here}: {p}" for p in _problems_for(value)]
101 else:
102 problems += check_json_urls(source, value, here)
103 elif isinstance(payload, list):
104 for i, item in enumerate(payload):
105 problems += check_json_urls(source, item, f"{_path}[{i}]")
106 return problems
107
108
109 def production_trusted_proxies(entrypoint: str, dockerfile: str = "") -> list[str]:
110 """``--forwarded-allow-ips`` in effect for each ``exec uvicorn`` command in the image entrypoint.
111
112 The flag wins; otherwise uvicorn reads ``FORWARDED_ALLOW_IPS`` (here: a Dockerfile ``ENV``);
113 otherwise it trusts only ``127.0.0.1``. Values set in a host's ``--env-file`` are not visible
114 to an offline check.
115 """
116 env = re.search(r"^\s*ENV\s+FORWARDED_ALLOW_IPS[=\s]+(['\"]?)([^\s'\"]+)\1", dockerfile, re.MULTILINE)
117 fallback = env.group(2) if env else UVICORN_DEFAULT_TRUSTED
118 joined = re.sub(r"\\\n", " ", entrypoint)
119 values: list[str] = []
120 for line in joined.splitlines():
121 if not re.match(r"^\s*exec\s+uvicorn\b", line):
122 continue
123 flag = re.search(r"--forwarded-allow-ips(?:=|\s+)(['\"]?)([^\s'\"]+)\1", line)
124 values.append(flag.group(2) if flag else fallback)
125 return values
File History 2 commits
sha256:eb10be5b0151404c767863448333d0cf0ad1b63219b5113abb902cb0132ae67e Merge dev (#242 DOC_INF_04) into task/243-docs-landing (#243) Opus 5.5 10 days ago
sha256:e8b5a50b0a352faa913ea7d5e72e615527564b52466009c56171d1d54f15b7ec test(docs): check that /muse pages served over HTTPS hand o… Opus 5.5 10 days ago