From 1cb74513ca35733452ced0a5f499324bf81d9467 Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Sun, 19 Jul 2026 19:20:49 +0530 Subject: [PATCH 01/10] fix: bound and validate remote JSON reads --- json2xml/utils.py | 83 +++++++++++++++++++++++++++++++++++++++++--- lat.md/behavior.md | 10 +++++- lat.md/tests.md | 8 +++++ tests/test_utils.py | 84 +++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 172 insertions(+), 13 deletions(-) diff --git a/json2xml/utils.py b/json2xml/utils.py index 2d24fb8..d8d0572 100644 --- a/json2xml/utils.py +++ b/json2xml/utils.py @@ -2,13 +2,17 @@ from __future__ import annotations import json +import socket +from ipaddress import ip_address from typing import Any +from urllib.parse import urlsplit __lazy_modules__ = ["urllib3"] from .types import JSONValue DEFAULT_URL_TIMEOUT: Any | None = None +DEFAULT_MAX_RESPONSE_BYTES = 10 * 1024 * 1024 _HTTP: Any | None = None @@ -54,9 +58,62 @@ def readfromjson(filename: str) -> JSONValue: raise JSONReadError("Invalid JSON File") from error -def readfromurl(url: str, params: dict[str, str] | None = None) -> JSONValue: +# @lat: [[behavior#URL security boundaries]] +def _validate_url(url: str, allow_private_networks: bool) -> None: + """Reject URL forms that can escape the intended public HTTP boundary.""" + try: + parsed = urlsplit(url) + port = parsed.port + except (TypeError, ValueError) as error: + raise URLReadError("URL is not valid") from error + + if parsed.scheme not in {"http", "https"}: + raise URLReadError("URL must use HTTP or HTTPS") + if parsed.username is not None or parsed.password is not None: + raise URLReadError("URL must not contain credentials") + if parsed.hostname is None: + raise URLReadError("URL must include a hostname") + if allow_private_networks: + return + + hostname = parsed.hostname + try: + addresses = {ip_address(hostname)} + except ValueError: + try: + address_info = socket.getaddrinfo( + hostname, + port or (443 if parsed.scheme == "https" else 80), + type=socket.SOCK_STREAM, + ) + except OSError as error: + raise URLReadError("URL hostname could not be resolved") from error + addresses = { + ip_address(info[4][0].split("%", 1)[0]) for info in address_info + } + + if not addresses or any(not address.is_global for address in addresses): + raise URLReadError("URL must resolve only to a public network address") + + +def readfromurl( + url: str, + params: dict[str, str] | None = None, + *, + max_response_bytes: int = DEFAULT_MAX_RESPONSE_BYTES, + allow_private_networks: bool = False, +) -> JSONValue: """Load JSON data from a URL.""" + if ( + isinstance(max_response_bytes, bool) + or not isinstance(max_response_bytes, int) + or max_response_bytes <= 0 + ): + raise URLReadError("Maximum response size must be a positive integer") + _validate_url(url, allow_private_networks) + urllib3, http, timeout = _get_http_client() + response = None try: response = http.request( "GET", @@ -64,15 +121,31 @@ def readfromurl(url: str, params: dict[str, str] | None = None) -> JSONValue: fields=params, timeout=timeout, retries=False, + redirect=False, + preload_content=False, ) + if response.status != 200: + raise URLReadError("URL is not returning correct response") + + content_length = response.headers.get("Content-Length") + if content_length is not None: + try: + if int(content_length) > max_response_bytes: + raise URLReadError("URL response exceeds maximum size") + except ValueError as error: + raise URLReadError("URL returned an invalid Content-Length") from error + + response_data = response.read(max_response_bytes + 1, decode_content=True) + if len(response_data) > max_response_bytes: + raise URLReadError("URL response exceeds maximum size") except urllib3.exceptions.HTTPError as error: raise URLReadError("URL could not be read") from error - - if response.status != 200: - raise URLReadError("URL is not returning correct response") + finally: + if response is not None: + response.close() try: - return json.loads(response.data.decode("utf-8")) + return json.loads(response_data.decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError) as error: raise URLReadError("URL did not return valid JSON") from error diff --git a/lat.md/behavior.md b/lat.md/behavior.md index 5dabd7d..e0e26e1 100644 --- a/lat.md/behavior.md +++ b/lat.md/behavior.md @@ -6,7 +6,15 @@ This file captures the observable conversion and input rules that matter more th The input helpers convert files, strings, URLs, and stdin into Python data structures while surfacing source-specific errors to callers. -[[json2xml/utils.py#readfromjson]] wraps file and JSON decoding failures in `JSONReadError`. [[json2xml/utils.py#readfromstring]] accepts unknown caller input so invalid-type tests can call it honestly, then rejects non-string inputs and malformed JSON with `StringReadError`. [[json2xml/utils.py#readfromurl]] lazily initializes the HTTP client, performs a bounded GET request, and raises `URLReadError` for network, non-200, decoding, and JSON parse failures. +[[json2xml/utils.py#readfromjson]] wraps file and JSON decoding failures in `JSONReadError`. [[json2xml/utils.py#readfromstring]] rejects non-string inputs and malformed JSON with `StringReadError`. + +[[json2xml/utils.py#readfromurl]] lazily initializes the HTTP client, performs a bounded GET request, and raises `URLReadError` for network, status, size, decoding, and JSON failures. + +## URL security boundaries + +Remote JSON reads default to public, credential-free HTTP(S) targets and bounded decoded responses so callers do not accidentally expose internal services or unlimited memory. + +[[json2xml/utils.py#readfromurl]] disables redirects, rejects non-global resolved addresses, and reads at most 10 MiB after content decoding. Trusted library callers can explicitly opt into private-network access while retaining the response limit. ## User examples diff --git a/lat.md/tests.md b/lat.md/tests.md index afd834b..d4cdb72 100644 --- a/lat.md/tests.md +++ b/lat.md/tests.md @@ -26,6 +26,14 @@ These tests verify the concrete reader helpers against realistic source behavior URL input should read valid JSON over HTTP and wrap status, network, and decoding failures in `URLReadError`. +### URL reader rejects unsafe destinations + +URL input should reject unsupported schemes, embedded credentials, and private or link-local targets unless a trusted library caller explicitly opts into private-network access. + +### URL reader limits decoded response size + +URL input should stop reading once the decoded response exceeds its configured limit so compressed or oversized remote content cannot exhaust process memory. + ## CLI failure messages These tests verify common command-line failures return short messages that name the broken input source and point users at the next valid action. diff --git a/tests/test_utils.py b/tests/test_utils.py index 8222acf..531ea17 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,6 +8,7 @@ from unittest.mock import Mock, patch import pytest +import urllib3 from json2xml.utils import ( InvalidDataError, @@ -151,31 +152,41 @@ class TestReadFromUrl: def test_readfromurl_success(self, json_server: str) -> None: """Test successful URL reading.""" - result = readfromurl(f"{json_server}/data.json") + result = readfromurl( + f"{json_server}/data.json", allow_private_networks=True + ) assert result == {"key": "value", "number": 42} def test_readfromurl_success_with_params(self, json_server: str) -> None: """Test successful URL reading with parameters.""" params = {"param1": "value1", "param2": "value2"} - result = readfromurl(f"{json_server}/api", params=params) + result = readfromurl( + f"{json_server}/api", params=params, allow_private_networks=True + ) assert result == {"result": "success"} def test_readfromurl_http_error(self, json_server: str) -> None: """Test URL reading with HTTP error status.""" with pytest.raises(URLReadError, match="URL is not returning correct response"): - readfromurl(f"{json_server}/nonexistent.json") + readfromurl( + f"{json_server}/nonexistent.json", allow_private_networks=True + ) def test_readfromurl_server_error(self, json_server: str) -> None: """Test URL reading with server error status.""" with pytest.raises(URLReadError, match="URL is not returning correct response"): - readfromurl(f"{json_server}/error.json") + readfromurl( + f"{json_server}/error.json", allow_private_networks=True + ) def test_readfromurl_invalid_json_response(self, json_server: str) -> None: """Test URL reading with invalid JSON response.""" with pytest.raises(URLReadError, match="URL did not return valid JSON"): - readfromurl(f"{json_server}/invalid.json") + readfromurl( + f"{json_server}/invalid.json", allow_private_networks=True + ) def test_readfromurl_network_error(self) -> None: """Test network failures are wrapped as URLReadError.""" @@ -184,7 +195,64 @@ def test_readfromurl_network_error(self) -> None: port = unused_socket.getsockname()[1] with pytest.raises(URLReadError, match="URL could not be read"): - readfromurl(f"http://127.0.0.1:{port}/data.json") + readfromurl( + f"http://127.0.0.1:{port}/data.json", allow_private_networks=True + ) + + # @lat: [[tests#Input readers#URL reader rejects unsafe destinations]] + def test_readfromurl_rejects_private_networks_by_default(self) -> None: + """Test URL reads cannot reach private or link-local services by default.""" + with pytest.raises(URLReadError, match="public network address"): + readfromurl("http://127.0.0.1/private.json") + + with pytest.raises(URLReadError, match="public network address"): + readfromurl("http://169.254.169.254/latest/meta-data/") + + @patch("json2xml.utils.socket.getaddrinfo") + def test_readfromurl_rejects_hostnames_resolving_to_private_networks( + self, mock_getaddrinfo: Mock + ) -> None: + """Test DNS names cannot bypass the private-network URL policy.""" + mock_getaddrinfo.return_value = [ + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.8", 443)) + ] + + with pytest.raises(URLReadError, match="public network address"): + readfromurl("https://internal.example/data.json") + + def test_readfromurl_rejects_unsupported_schemes_and_credentials(self) -> None: + """Test URL reads accept only credential-free HTTP and HTTPS URLs.""" + with pytest.raises(URLReadError, match="HTTP or HTTPS"): + readfromurl("file:///etc/passwd") + + with pytest.raises(URLReadError, match="credentials"): + readfromurl("https://user:password@8.8.8.8/data.json") + + @patch("json2xml.utils._get_http_client") + # @lat: [[tests#Input readers#URL reader limits decoded response size]] + def test_readfromurl_limits_decoded_response_size( + self, mock_get_http_client: Mock + ) -> None: + """Test URL reads stop after the configured decoded-byte limit.""" + response = Mock(status=200, headers={}) + response.read.return_value = b'{"value":"payload larger than limit"}' + http = Mock() + http.request.return_value = response + mock_get_http_client.return_value = (urllib3, http, Mock()) + + with pytest.raises(URLReadError, match="maximum size"): + readfromurl("https://8.8.8.8/data.json", max_response_bytes=16) + + http.request.assert_called_once_with( + "GET", + "https://8.8.8.8/data.json", + fields=None, + timeout=mock_get_http_client.return_value[2], + retries=False, + redirect=False, + preload_content=False, + ) + response.close.assert_called_once_with() class TestReadFromString: @@ -284,7 +352,9 @@ def test_readfromurl_then_convert_to_xml(self, json_server: str) -> None: """Test reading from URL and converting to XML.""" from json2xml import dicttoxml - data = readfromurl(f"{json_server}/api.json") + data = readfromurl( + f"{json_server}/api.json", allow_private_networks=True + ) xml_result = dicttoxml.dicttoxml(data, attr_type=False, root=False) assert b"response" in xml_result From b4f7b533cd886c497be84c092718bdd27d121bfb Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Sun, 19 Jul 2026 19:30:42 +0530 Subject: [PATCH 02/10] fix: reject unsafe XML output --- json2xml/dicttoxml.py | 71 +++++++++++++++++++------- lat.md/architecture.md | 2 + lat.md/behavior.md | 6 +++ lat.md/tests.md | 12 +++++ rust/src/lib.rs | 33 ++++++++++-- tests/test_dict2xml.py | 99 ++++++++++++++++++++++++++++++++++++ tests/test_rust_dicttoxml.py | 14 +++++ 7 files changed, 215 insertions(+), 22 deletions(-) diff --git a/json2xml/dicttoxml.py b/json2xml/dicttoxml.py index d1c46b6..e9962e9 100644 --- a/json2xml/dicttoxml.py +++ b/json2xml/dicttoxml.py @@ -2,6 +2,7 @@ import datetime import numbers +import re from collections.abc import Callable, Sequence from dataclasses import dataclass from decimal import Decimal @@ -17,6 +18,9 @@ _SAFE_RANDOM = SystemRandom() _XML_ESCAPE_CHARS = frozenset("&\"'<>") +_XML_INVALID_CHAR_RE = re.compile( + "[\x00-\x08\x0b\x0c\x0e-\x1f\ud800-\udfff\ufffe\uffff]" +) class _XMLWriter: @@ -125,6 +129,17 @@ def get_xml_type(val: Any) -> str: return type(val).__name__ +# @lat: [[behavior#XML output safety]] +def _validate_xml_chars(value: str) -> None: + """Reject characters excluded from the XML 1.0 Char production.""" + invalid_match = _XML_INVALID_CHAR_RE.search(value) + if invalid_match is not None: + codepoint = ord(invalid_match.group()) + raise ValueError( + f"Character U+{codepoint:04X} is not allowed in XML 1.0" + ) + + def escape_xml(s: str | int | float | numbers.Number | None) -> str: """ Escape a string for use in XML. @@ -135,15 +150,15 @@ def escape_xml(s: str | int | float | numbers.Number | None) -> str: Returns: str: The escaped string. """ - if isinstance(s, str): - if _XML_ESCAPE_CHARS.isdisjoint(s): - return s - s = s.replace("&", "&") - s = s.replace('"', """) - s = s.replace("'", "'") - s = s.replace("<", "<") - s = s.replace(">", ">") - return str(s) + value = s if isinstance(s, str) else str(s) + _validate_xml_chars(value) + if _XML_ESCAPE_CHARS.isdisjoint(value): + return value + value = value.replace("&", "&") + value = value.replace('"', """) + value = value.replace("'", "'") + value = value.replace("<", "<") + return value.replace(">", ">") def make_attrstring(attr: dict[str, Any]) -> str: @@ -161,7 +176,7 @@ def make_attrstring(attr: dict[str, Any]) -> str: if len(attr) == 1: key, val = next(iter(attr.items())) if key == "type": - return f' type="{val}"' + return f' type="{escape_xml(val)}"' validate_xml_attr_names(attr) return f' {key}="{escape_xml(val)}"' validate_xml_attr_names(attr) @@ -263,8 +278,10 @@ def make_valid_xml_name(key: str, attr: dict[str, Any]) -> tuple[str, dict[str, def wrap_cdata(s: str | int | float | numbers.Number) -> str: """Wraps a string into CDATA sections""" - s = str(s).replace("]]>", "]]]]>") - return "" + value = str(s) + _validate_xml_chars(value) + value = value.replace("]]>", "]]]]>") + return "" def default_item_func(parent: str) -> str: @@ -1113,6 +1130,7 @@ def render(self) -> bytes: return output.to_bytes() +# @lat: [[behavior#XML output safety]] class _NamespaceFormatter: """Keep namespace emission and schema-attribute quirks in one place.""" @@ -1122,21 +1140,38 @@ def format(xml_namespaces: dict[str, Any] | None) -> str: return "" namespace_parts: list[str] = [] - for prefix in xml_namespaces: + for prefix, namespace_value in xml_namespaces.items(): if prefix == "xsi": - for schema_att in xml_namespaces[prefix]: + if not isinstance(namespace_value, dict): + raise ValueError("The xsi namespace value must be a mapping") + if ( + "schemaLocation" in namespace_value + and "schemaInstance" not in namespace_value + ): + raise ValueError( + "xsi schemaLocation requires a schemaInstance namespace" + ) + for schema_att in namespace_value: if schema_att == "schemaInstance": namespace_parts.append( - f' xmlns:{prefix}="{xml_namespaces[prefix]["schemaInstance"]}"' + f' xmlns:{prefix}="{escape_xml(namespace_value[schema_att])}"' ) elif schema_att == "schemaLocation": namespace_parts.append( - f' xsi:{schema_att}="{xml_namespaces[prefix][schema_att]}"' + f' xsi:{schema_att}="{escape_xml(namespace_value[schema_att])}"' ) elif prefix == "xmlns": - namespace_parts.append(f' xmlns="{xml_namespaces[prefix]}"') + namespace_parts.append(f' xmlns="{escape_xml(namespace_value)}"') else: - namespace_parts.append(f' xmlns:{prefix}="{xml_namespaces[prefix]}"') + if ( + not isinstance(prefix, str) + or prefix.lower().startswith("xml") + or not key_is_valid_xml(prefix) + ): + raise ValueError(f"Invalid XML namespace prefix: {prefix}") + namespace_parts.append( + f' xmlns:{prefix}="{escape_xml(namespace_value)}"' + ) return "".join(namespace_parts) diff --git a/lat.md/architecture.md b/lat.md/architecture.md index b33f30e..960732c 100644 --- a/lat.md/architecture.md +++ b/lat.md/architecture.md @@ -18,6 +18,8 @@ The `dicttoxml()` entry point now normalizes options into `SerializerConfig` and The recursive serializer still streams normal and XPath serialization through [[json2xml/dicttoxml.py#_XMLWriter]] so dict and list payloads do not allocate a complete string for each nested subtree. Public helpers such as `convert_dict()` still return strings for compatibility by delegating to the same append path, while library and CLI conversions write UTF-8 bytes incrementally and return the final `bytes` object. Attribute formatting stays centralized through `make_attrstring()`, and `@attrs`/`@val` normalization stays local to dict element handling so caller-owned metadata is never mutated. +Text, CDATA, custom attributes, and namespace declarations share XML 1.0 character validation. Namespace declarations additionally validate prefixes before the renderer appends them to the root element. + ## Backend selection The fast-path module prefers the Rust extension when it can preserve Python semantics, and falls back to the Python serializer for unsupported features. diff --git a/lat.md/behavior.md b/lat.md/behavior.md index e0e26e1..9c3afba 100644 --- a/lat.md/behavior.md +++ b/lat.md/behavior.md @@ -43,3 +43,9 @@ When `xpath_format=True`, [[json2xml/dicttoxml.py#dicttoxml]] delegates payload Pretty printing acts as a validation step, because the formatter reparses the generated XML before returning it. [[json2xml/json2xml.py#Json2xml#to_xml]] imports `defusedxml.minidom.parseString` only for pretty output, then reparses before `toprettyxml`. If the generated bytes are not well-formed XML, the converter raises `InvalidDataError` instead of returning broken pretty output. + +## XML output safety + +Every serializer mode rejects XML 1.0-forbidden characters and treats namespace metadata as attributes so raw output cannot bypass well-formedness or escaping checks. + +[[json2xml/dicttoxml.py#escape_xml]] and CDATA writers reject forbidden control characters before output. Namespace prefixes are validated, while namespace and custom attribute values use the shared XML escaping path. diff --git a/lat.md/tests.md b/lat.md/tests.md index d4cdb72..28af716 100644 --- a/lat.md/tests.md +++ b/lat.md/tests.md @@ -90,6 +90,10 @@ Supplying namespace prefixes and an `xsi` mapping should emit the expected `xmln Reusing one `xml_namespaces` mapping across multiple `dicttoxml` calls should return identical XML each time so namespace declarations never accumulate on the shared dict. +### Namespace metadata cannot inject attributes + +Namespace prefixes must be valid names and namespace values must be escaped as attribute values so quotes cannot create attacker-controlled root attributes. + ### Falsy JSON values convert to XML Falsy JSON values such as empty objects, empty arrays, zero, false, and empty strings should convert through the public API instead of being treated as missing data. @@ -166,6 +170,14 @@ Invalid custom root names should use the serializer's existing XML-name normaliz Custom `@attrs` keys that are not valid XML attribute names should fail explicitly because attributes have no safe metadata fallback equivalent to element `` output. +### Custom type attributes use shared escaping + +Caller-provided `type` attributes must use the same XML value escaping as every other custom attribute instead of taking the internal type fast path. + +### XML 1.0 forbidden characters are rejected + +Text and CDATA output must reject forbidden XML 1.0 control characters before raw bytes are returned, while preserving valid tab, newline, and carriage-return characters. + ## XML helper behavior These tests pin low-level XML helper contracts so performance refactors keep the same serializer output and caller-side mutation behavior. diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 8d1f8f7..ff0d104 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -19,6 +19,27 @@ const OUTPUT_BUFFER_SIZE: usize = 16 * 1024; const SPARSE_ESCAPE_SCAN_LIMIT: u8 = 4; +#[inline] +fn invalid_xml_char(s: &str) -> Option { + s.chars().find(|character| { + let codepoint = u32::from(*character); + !matches!(codepoint, 0x9 | 0xA | 0xD | 0x20..=0xD7FF | 0xE000..=0xFFFD | 0x10000..=0x10FFFF) + }) +} + +// @lat: [[behavior#XML output safety]] +#[cfg(feature = "python")] +#[inline] +fn validate_xml_chars(s: &str) -> PyResult<()> { + if let Some(character) = invalid_xml_char(s) { + return Err(PyValueError::new_err(format!( + "Character U+{:04X} is not allowed in XML 1.0", + u32::from(character) + ))); + } + Ok(()) +} + /// Return the byte offset of the next character requiring XML escaping. #[inline(always)] fn next_xml_escape(bytes: &[u8]) -> Option { @@ -116,6 +137,7 @@ fn write_byte(out: &mut W, b: u8) -> PyResult<()> { #[cfg(feature = "python")] #[inline] fn write_escaped_text(out: &mut W, s: &str) -> PyResult<()> { + validate_xml_chars(s)?; let bytes = s.as_bytes(); let mut last = 0; for _ in 0..SPARSE_ESCAPE_SCAN_LIMIT { @@ -147,6 +169,7 @@ fn write_escaped_attr(out: &mut W, s: &str) -> PyResult<()> { #[cfg(feature = "python")] #[inline] fn write_cdata(out: &mut W, s: &str) -> PyResult<()> { + validate_xml_chars(s)?; write_str(out, "") { @@ -592,15 +615,17 @@ fn dicttoxml( /// Escapes &, ", ', <, > characters for XML. #[cfg(feature = "python")] #[pyfunction] -fn escape_xml_py(s: &str) -> String { - escape_xml(s) +fn escape_xml_py(s: &str) -> PyResult { + validate_xml_chars(s)?; + Ok(escape_xml(s)) } /// Wrap a string in CDATA section. #[cfg(feature = "python")] #[pyfunction] -fn wrap_cdata_py(s: &str) -> String { - wrap_cdata(s) +fn wrap_cdata_py(s: &str) -> PyResult { + validate_xml_chars(s)?; + Ok(wrap_cdata(s)) } /// A Python module implemented in Rust. diff --git a/tests/test_dict2xml.py b/tests/test_dict2xml.py index 2332918..ad726ac 100644 --- a/tests/test_dict2xml.py +++ b/tests/test_dict2xml.py @@ -89,6 +89,64 @@ def test_dict2xml_xsi_xmlns(self) -> None: "blue" == result ) + # @lat: [[tests#Conversion behavior#Namespace metadata cannot inject attributes]] + def test_namespace_values_are_escaped_as_xml_attributes(self) -> None: + """Namespace values must not be able to inject sibling attributes.""" + namespaces = { + "veh": 'urn:vehicle" injected="yes', + "xmlns": 'urn:default" default-injected="yes', + "xsi": { + "schemaInstance": 'urn:xsi" xsi-injected="yes', + "schemaLocation": 'vehicle.xsd" location-injected="yes', + }, + } + + result = dicttoxml.dicttoxml( + {"bike": "blue"}, attr_type=False, xml_namespaces=namespaces + ) + + assert b' injected="yes"' not in result + assert b' default-injected="yes"' not in result + assert b' xsi-injected="yes"' not in result + assert b' location-injected="yes"' not in result + assert b'xmlns:veh="urn:vehicle" injected="yes"' in result + assert b'xmlns="urn:default" default-injected="yes"' in result + assert b'xmlns:xsi="urn:xsi" xsi-injected="yes"' in result + assert ( + b'xsi:schemaLocation="vehicle.xsd" location-injected="yes"' + in result + ) + + @pytest.mark.parametrize( + "prefix", [1, "xmlData", "bad prefix", 'bad" injected="yes'] + ) + def test_invalid_namespace_prefixes_are_rejected(self, prefix: Any) -> None: + """Namespace prefixes must be valid XML namespace names.""" + with pytest.raises(ValueError, match="Invalid XML namespace prefix"): + dicttoxml.dicttoxml( + {"bike": "blue"}, + attr_type=False, + xml_namespaces={prefix: "urn:vehicle"}, + ) + + @pytest.mark.parametrize( + ("xsi_value", "message"), + [ + ("urn:xsi", "must be a mapping"), + ({"schemaLocation": "vehicle.xsd"}, "requires a schemaInstance"), + ], + ) + def test_invalid_xsi_namespace_shapes_are_rejected( + self, xsi_value: Any, message: str + ) -> None: + """XSI schema attributes require a well-formed namespace mapping.""" + with pytest.raises(ValueError, match=message): + dicttoxml.dicttoxml( + {"bike": "blue"}, + attr_type=False, + xml_namespaces={"xsi": xsi_value}, + ) + # @lat: [[tests#Conversion behavior#XPath format wraps root scalars]] def test_xpath_format_root_scalar_wraps_in_namespace_map(self) -> None: """Test XPath root scalar output remains one namespace-qualified document.""" @@ -539,6 +597,47 @@ def test_dicttoxml_accepts_valid_custom_attribute_edge_names(self, attr_name: st assert result == f'payload'.encode() + # @lat: [[tests#Conversion behavior#Custom type attributes use shared escaping]] + def test_custom_type_attribute_value_is_escaped(self) -> None: + """A caller-provided type attribute must not bypass value escaping.""" + result = dicttoxml.dicttoxml( + { + "key": { + "@attrs": {"type": 'safe" injected="yes'}, + "@val": "payload", + } + }, + root=False, + attr_type=False, + ) + + assert result == ( + b'payload' + ) + + @pytest.mark.parametrize("invalid_char", ["\x00", "\x01", "\x0b", "\x1b", "\ufffe"]) + @pytest.mark.parametrize("cdata", [False, True]) + # @lat: [[tests#Conversion behavior#XML 1.0 forbidden characters are rejected]] + def test_dicttoxml_rejects_xml_1_0_forbidden_characters( + self, invalid_char: str, cdata: bool + ) -> None: + """XML 1.0-forbidden controls must not reach raw text or CDATA output.""" + with pytest.raises(ValueError, match="not allowed in XML 1.0"): + dicttoxml.dicttoxml( + {"key": f"before{invalid_char}after"}, + root=False, + attr_type=False, + cdata=cdata, + ) + + def test_dicttoxml_preserves_xml_1_0_whitespace_characters(self) -> None: + """Tabs, newlines, and carriage returns remain valid XML text.""" + result = dicttoxml.dicttoxml( + {"key": "tab\tline\nreturn\r"}, root=False, attr_type=False + ) + + assert result == b"tab\tline\nreturn\r" + def test_dicttoxml_with_xml_namespaces(self) -> None: """Test dicttoxml with XML namespaces.""" data = {"key": "value"} diff --git a/tests/test_rust_dicttoxml.py b/tests/test_rust_dicttoxml.py index 1e8de56..3dbd824 100644 --- a/tests/test_rust_dicttoxml.py +++ b/tests/test_rust_dicttoxml.py @@ -68,6 +68,11 @@ def test_escape_no_special_chars(self): def test_escape_unicode(self): assert escape_xml_py("héllo wörld 日本語") == "héllo wörld 日本語" + @pytest.mark.parametrize("invalid_char", ["\x00", "\x01", "\x0b", "\x1b", "\ufffe"]) + def test_escape_rejects_xml_1_0_forbidden_characters(self, invalid_char: str): + with pytest.raises(ValueError, match="not allowed in XML 1.0"): + escape_xml_py(f"before{invalid_char}after") + class TestRustWrapCdata: """Test the Rust wrap_cdata function.""" @@ -86,6 +91,10 @@ def test_wrap_with_cdata_end(self): def test_wrap_empty(self): assert wrap_cdata_py("") == "" + def test_wrap_rejects_xml_1_0_forbidden_characters(self): + with pytest.raises(ValueError, match="not allowed in XML 1.0"): + wrap_cdata_py("before\x1bafter") + class TestRustDicttoxml: """Test the main Rust dicttoxml function.""" @@ -107,6 +116,11 @@ def test_string_value(self): assert b">Hello World" in result assert b'type="str"' in result + @pytest.mark.parametrize("cdata", [False, True]) + def test_string_value_rejects_xml_1_0_forbidden_characters(self, cdata: bool): + with pytest.raises(ValueError, match="not allowed in XML 1.0"): + rust_dicttoxml({"message": "before\x01after"}, cdata=cdata) + def test_integer_value(self): data = {"count": 42} result = rust_dicttoxml(data) From 0145f16051f32a985229d1398d26bea63abe1ced Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Sun, 19 Jul 2026 19:37:32 +0530 Subject: [PATCH 03/10] chore: sync Python security dependencies --- .github/dependabot.yml | 8 ++++++++ lat.md/architecture.md | 2 ++ requirements.txt | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ca79ca5..8904b39 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,11 @@ updates: directory: / schedule: interval: weekly + - package-ecosystem: pip + directory: / + schedule: + interval: weekly + - package-ecosystem: pip + directory: /docs + schedule: + interval: weekly diff --git a/lat.md/architecture.md b/lat.md/architecture.md index 960732c..054f1e7 100644 --- a/lat.md/architecture.md +++ b/lat.md/architecture.md @@ -70,6 +70,8 @@ Dependency floors and lockfiles keep known vulnerable packages out of runtime an Runtime dependencies are declared in `pyproject.toml` and mirrored by `uv.lock`; legacy requirements inputs remain pinned for tooling that still consumes requirements files. Security fixes should update both resolver paths so `uv audit` and requirements-based installs agree. +Dependabot checks the root and documentation Python dependency manifests weekly, alongside the existing GitHub Actions monitoring, so stale security pins are surfaced automatically. + ## Workflow supply-chain hardening GitHub Actions workflows run with read-only tokens by default and use full SHA pins so third-party action updates are explicit. diff --git a/requirements.txt b/requirements.txt index 1d61858..c64650f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,5 @@ # defusedxml==0.7.1 # via -r requirements.in -urllib3==2.5.0 +urllib3==2.7.0 # via -r requirements.in From 980ec347d73112d1768e5cfa446622e6353be311 Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Sun, 19 Jul 2026 19:41:14 +0530 Subject: [PATCH 04/10] fix: disable unsafe Rust accelerators --- json2xml/dicttoxml_fast.py | 21 +++++++++++++++++++-- lat.md/architecture.md | 2 +- lat.md/tests.md | 4 ++++ tests/test_dicttoxml_fast_fallback.py | 16 ++++++++++++++++ 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/json2xml/dicttoxml_fast.py b/json2xml/dicttoxml_fast.py index da01ab4..fcfc39c 100644 --- a/json2xml/dicttoxml_fast.py +++ b/json2xml/dicttoxml_fast.py @@ -30,12 +30,29 @@ rust_escape_xml: RustStringTransform | None = None rust_wrap_cdata: RustStringTransform | None = None + +def _rejects_invalid_xml(escape: RustStringTransform) -> bool: + """Return whether an optional backend enforces XML 1.0 characters.""" + try: + escape("\x00") + except ValueError: + return True + except Exception: + return False + return False + + try: from json2xml_rs import dicttoxml as _rust_dicttoxml # pragma: no cover from json2xml_rs import escape_xml_py as rust_escape_xml # pragma: no cover from json2xml_rs import wrap_cdata_py as rust_wrap_cdata # pragma: no cover - _use_rust = True # pragma: no cover - LOG.debug("Using Rust backend for dicttoxml") # pragma: no cover + if _rejects_invalid_xml(rust_escape_xml): # pragma: no cover + _use_rust = True # pragma: no cover + LOG.debug("Using Rust backend for dicttoxml") # pragma: no cover + else: # pragma: no cover + LOG.warning( # pragma: no cover + "Ignoring an outdated Rust backend that permits invalid XML characters" + ) except ImportError: # pragma: no cover LOG.debug("Rust backend not available, using pure Python") diff --git a/lat.md/architecture.md b/lat.md/architecture.md index 054f1e7..5f96130 100644 --- a/lat.md/architecture.md +++ b/lat.md/architecture.md @@ -24,7 +24,7 @@ Text, CDATA, custom attributes, and namespace declarations share XML 1.0 charact The fast-path module prefers the Rust extension when it can preserve Python semantics, and falls back to the Python serializer for unsupported features. -[[json2xml/dicttoxml_fast.py#dicttoxml]] now normalizes each call into a shared conversion request and asks a tiny backend selector seam to choose Rust or Python. The Rust adapter accepts only requests whose semantics it can preserve, namely no `ids`, custom `item_func`, XML namespaces, XPath mode, root scalar payloads, or special `@` keys. +[[json2xml/dicttoxml_fast.py#dicttoxml]] now normalizes each call into a shared conversion request and asks a tiny backend selector seam to choose Rust or Python. The Rust adapter accepts only requests whose semantics it can preserve, namely no `ids`, custom `item_func`, XML namespaces, XPath mode, root scalar payloads, or special `@` keys. At import time, the wrapper also verifies that an installed Rust backend rejects XML 1.0 forbidden characters; outdated or broken accelerators stay disabled so the Python security boundary cannot be bypassed. The backend adapter protocol exposes its diagnostic name as a read-only property, matching the frozen adapter implementations while still allowing selector code to inspect backend metadata. diff --git a/lat.md/tests.md b/lat.md/tests.md index 28af716..b64d7a5 100644 --- a/lat.md/tests.md +++ b/lat.md/tests.md @@ -66,6 +66,10 @@ The multi-interpreter benchmark should let per-interpreter environment variables These tests pin the XML shapes that matter most for interoperability, especially the modes that intentionally diverge from the default serializer. +### Outdated Rust backends stay disabled + +An optional Rust accelerator is eligible only when its escape helper rejects XML 1.0 forbidden characters, preventing an older wheel from bypassing the Python serializer's validation. + ### XPath format adds functions namespace XPath mode should emit the W3C XPath functions namespace and typed child elements so downstream consumers receive standards-shaped XML. diff --git a/tests/test_dicttoxml_fast_fallback.py b/tests/test_dicttoxml_fast_fallback.py index 7e7a0f1..d2381ac 100644 --- a/tests/test_dicttoxml_fast_fallback.py +++ b/tests/test_dicttoxml_fast_fallback.py @@ -17,6 +17,22 @@ def _force_rust_backend(monkeypatch: pytest.MonkeyPatch) -> Mock: return rust_backend +# @lat: [[tests#Conversion behavior#Outdated Rust backends stay disabled]] +@pytest.mark.parametrize( + ("escape", "expected"), + [ + (Mock(return_value="\x00"), False), + (Mock(side_effect=RuntimeError("broken backend")), False), + (Mock(side_effect=ValueError("invalid XML")), True), + ], +) +def test_rust_backend_must_reject_invalid_xml( + escape: Mock, expected: bool +) -> None: + """Only backends that reject XML 1.0 control characters are safe to use.""" + assert fast_module._rejects_invalid_xml(escape) is expected + + # @lat: [[tests#Conversion behavior#Fast wrapper exposes backend metadata]] def test_fast_wrapper_reports_python_backend_when_rust_is_unavailable( monkeypatch: pytest.MonkeyPatch, From a73ba8526850afd62e0c3fde8e545807eb0af5e1 Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Sun, 19 Jul 2026 19:41:14 +0530 Subject: [PATCH 05/10] docs: explain secure URL reader defaults --- README.rst | 12 ++++++++++++ docs/usage.rst | 13 +++++++++++++ json2xml/utils.py | 6 +++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a44804c..f32125c 100644 --- a/README.rst +++ b/README.rst @@ -239,6 +239,18 @@ You can use the json2xml library in the following ways: data = readfromjson("examples/licht.json") print(json2xml.Json2xml(data).to_xml()) +URL reads accept only credential-free HTTP(S), reject redirects and non-public +destinations by default, and stop after 10 MiB of decoded content. Trusted +library callers can opt into a private endpoint or choose a smaller limit: + +.. code-block:: python + + data = readfromurl( + "http://127.0.0.1:8000/data.json", + allow_private_networks=True, + max_response_bytes=1024 * 1024, + ) + Custom Wrappers and Indentation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/usage.rst b/docs/usage.rst index 6185bfa..023c2f0 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -32,6 +32,19 @@ Here's how to use each method: data = readfromjson("examples/licht.json") print(json2xml.Json2xml(data).to_xml()) +URL reads accept only credential-free HTTP(S), reject redirects and non-public +destinations by default, and stop after 10 MiB of decoded content. Trusted +library callers can explicitly allow a private endpoint while retaining a +chosen response limit: + +.. code-block:: python + + data = readfromurl( + "http://127.0.0.1:8000/data.json", + allow_private_networks=True, + max_response_bytes=1024 * 1024, + ) + Real-world Examples ------------------- diff --git a/json2xml/utils.py b/json2xml/utils.py index d8d0572..a3fae39 100644 --- a/json2xml/utils.py +++ b/json2xml/utils.py @@ -103,7 +103,11 @@ def readfromurl( max_response_bytes: int = DEFAULT_MAX_RESPONSE_BYTES, allow_private_networks: bool = False, ) -> JSONValue: - """Load JSON data from a URL.""" + """Load bounded JSON data from a public URL. + + Private-network access is available only through the explicit trusted-caller + opt-in. Redirects and embedded credentials are always rejected. + """ if ( isinstance(max_response_bytes, bool) or not isinstance(max_response_bytes, int) From fdb8f5ddd2e74c73d0cb9f92c33da618122aee36 Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Sun, 19 Jul 2026 19:47:10 +0530 Subject: [PATCH 06/10] test: cover hardened failure paths --- json2xml/json2xml.py | 23 ++++++++++++---------- json2xml/utils.py | 3 ++- tests/test_json2xml.py | 11 +++++++++++ tests/test_utils.py | 44 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 11 deletions(-) diff --git a/json2xml/json2xml.py b/json2xml/json2xml.py index 14e339d..8955214 100644 --- a/json2xml/json2xml.py +++ b/json2xml/json2xml.py @@ -41,16 +41,19 @@ def to_xml(self) -> Any | None: Convert to xml using dicttoxml.dicttoxml and then pretty print it. """ if self.data is not None: - xml_data = dicttoxml.dicttoxml( - self.data, - root=self.root, - custom_root=self.wrapper, - attr_type=self.attr_type, - item_wrap=self.item_wrap, - xpath_format=self.xpath_format, - cdata=self.cdata, - list_headers=self.list_headers, - ) + try: + xml_data = dicttoxml.dicttoxml( + self.data, + root=self.root, + custom_root=self.wrapper, + attr_type=self.attr_type, + item_wrap=self.item_wrap, + xpath_format=self.xpath_format, + cdata=self.cdata, + list_headers=self.list_headers, + ) + except ValueError as error: + raise InvalidDataError from error if self.pretty: from pyexpat import ExpatError diff --git a/json2xml/utils.py b/json2xml/utils.py index a3fae39..1284bf6 100644 --- a/json2xml/utils.py +++ b/json2xml/utils.py @@ -89,7 +89,8 @@ def _validate_url(url: str, allow_private_networks: bool) -> None: except OSError as error: raise URLReadError("URL hostname could not be resolved") from error addresses = { - ip_address(info[4][0].split("%", 1)[0]) for info in address_info + ip_address(str(info[4][0]).split("%", 1)[0]) + for info in address_info } if not addresses or any(not address.is_global for address in addresses): diff --git a/tests/test_json2xml.py b/tests/test_json2xml.py index 20004ab..03f76f3 100644 --- a/tests/test_json2xml.py +++ b/tests/test_json2xml.py @@ -4,6 +4,7 @@ from pyexpat import ExpatError from typing import Any +from unittest.mock import Mock import pytest import xmltodict @@ -208,6 +209,16 @@ def test_bad_data(self) -> None: json2xml.Json2xml({"bad": decoded}).to_xml() assert pytest_wrapped_e.type == InvalidDataError + def test_pretty_print_parser_errors_are_wrapped( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Parser failures preserve the public InvalidDataError contract.""" + parse_string = Mock(side_effect=ExpatError("malformed XML")) + monkeypatch.setattr("defusedxml.minidom.parseString", parse_string) + + with pytest.raises(InvalidDataError): + json2xml.Json2xml({"valid": "data"}).to_xml() + def test_read_boolean_data_from_json(self) -> None: """Test correct return for boolean types.""" data = readfromjson("examples/booleanjson.json") diff --git a/tests/test_utils.py b/tests/test_utils.py index 531ea17..abbe36a 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -228,6 +228,50 @@ def test_readfromurl_rejects_unsupported_schemes_and_credentials(self) -> None: with pytest.raises(URLReadError, match="credentials"): readfromurl("https://user:password@8.8.8.8/data.json") + with pytest.raises(URLReadError, match="include a hostname"): + readfromurl("https:///data.json") + + with pytest.raises(URLReadError, match="not valid"): + readfromurl("https://8.8.8.8:not-a-port/data.json") + + @patch("json2xml.utils.socket.getaddrinfo") + def test_readfromurl_rejects_unresolvable_hostnames( + self, mock_getaddrinfo: Mock + ) -> None: + """Test DNS failures are reported without attempting an HTTP request.""" + mock_getaddrinfo.side_effect = OSError("DNS unavailable") + + with pytest.raises(URLReadError, match="could not be resolved"): + readfromurl("https://unresolvable.example/data.json") + + def test_readfromurl_rejects_invalid_response_limit(self) -> None: + """Test callers cannot disable the response cap with a non-positive value.""" + with pytest.raises(URLReadError, match="positive integer"): + readfromurl("https://8.8.8.8/data.json", max_response_bytes=0) + + @pytest.mark.parametrize( + ("content_length", "message"), + [("17", "maximum size"), ("not-a-number", "invalid Content-Length")], + ) + @patch("json2xml.utils._get_http_client") + def test_readfromurl_rejects_invalid_content_lengths( + self, + mock_get_http_client: Mock, + content_length: str, + message: str, + ) -> None: + """Test declared response sizes are validated before reading the body.""" + response = Mock(status=200, headers={"Content-Length": content_length}) + http = Mock() + http.request.return_value = response + mock_get_http_client.return_value = (urllib3, http, Mock()) + + with pytest.raises(URLReadError, match=message): + readfromurl("https://8.8.8.8/data.json", max_response_bytes=16) + + response.read.assert_not_called() + response.close.assert_called_once_with() + @patch("json2xml.utils._get_http_client") # @lat: [[tests#Input readers#URL reader limits decoded response size]] def test_readfromurl_limits_decoded_response_size( From 36b3a11edf070fb3b664be64d323d03bcf62b46a Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Sun, 19 Jul 2026 20:01:02 +0530 Subject: [PATCH 07/10] docs: explain XML invalid character ranges --- json2xml/dicttoxml.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/json2xml/dicttoxml.py b/json2xml/dicttoxml.py index e9962e9..e51cc11 100644 --- a/json2xml/dicttoxml.py +++ b/json2xml/dicttoxml.py @@ -18,6 +18,10 @@ _SAFE_RANDOM = SystemRandom() _XML_ESCAPE_CHARS = frozenset("&\"'<>") +# XML 1.0 permits tab (U+0009), line feed (U+000A), carriage return +# (U+000D), and U+0020 onward, except for surrogate code points and the +# U+FFFE/U+FFFF noncharacters. This pattern matches only the forbidden ranges: +# disallowed C0 controls, UTF-16 surrogates, and the two BMP noncharacters. _XML_INVALID_CHAR_RE = re.compile( "[\x00-\x08\x0b\x0c\x0e-\x1f\ud800-\udfff\ufffe\uffff]" ) From 9017e034446766555e5e7b613fd63f070d793fc2 Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Sun, 19 Jul 2026 20:04:47 +0530 Subject: [PATCH 08/10] docs: link XML character specification --- json2xml/dicttoxml.py | 1 + 1 file changed, 1 insertion(+) diff --git a/json2xml/dicttoxml.py b/json2xml/dicttoxml.py index e51cc11..14bfb65 100644 --- a/json2xml/dicttoxml.py +++ b/json2xml/dicttoxml.py @@ -18,6 +18,7 @@ _SAFE_RANDOM = SystemRandom() _XML_ESCAPE_CHARS = frozenset("&\"'<>") +# XML 1.0 character production: https://www.w3.org/TR/xml/#charsets # XML 1.0 permits tab (U+0009), line feed (U+000A), carriage return # (U+000D), and U+0020 onward, except for surrogate code points and the # U+FFFE/U+FFFF noncharacters. This pattern matches only the forbidden ranges: From d736794ac839a91426c796e5866868ebbe4d6624 Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Sun, 19 Jul 2026 20:15:11 +0530 Subject: [PATCH 09/10] test: validate custom type attribute characters --- lat.md/tests.md | 4 ++++ tests/test_dict2xml.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/lat.md/tests.md b/lat.md/tests.md index b64d7a5..1a3527f 100644 --- a/lat.md/tests.md +++ b/lat.md/tests.md @@ -178,6 +178,10 @@ Custom `@attrs` keys that are not valid XML attribute names should fail explicit Caller-provided `type` attributes must use the same XML value escaping as every other custom attribute instead of taking the internal type fast path. +### Custom type attributes are validated + +Caller-provided `type` attribute values containing XML 1.0-forbidden characters must fail before serialization so the internal type fast path cannot bypass character validation. + ### XML 1.0 forbidden characters are rejected Text and CDATA output must reject forbidden XML 1.0 control characters before raw bytes are returned, while preserving valid tab, newline, and carriage-return characters. diff --git a/tests/test_dict2xml.py b/tests/test_dict2xml.py index ad726ac..c6927c5 100644 --- a/tests/test_dict2xml.py +++ b/tests/test_dict2xml.py @@ -615,6 +615,21 @@ def test_custom_type_attribute_value_is_escaped(self) -> None: b'payload' ) + # @lat: [[tests#Conversion behavior#Custom type attributes are validated]] + def test_custom_type_attribute_forbidden_chars_are_rejected(self) -> None: + """Forbidden XML characters in caller-provided type attributes fail.""" + with pytest.raises(ValueError, match="not allowed in XML 1.0"): + dicttoxml.dicttoxml( + { + "key": { + "@attrs": {"type": "before\x00after"}, + "@val": "payload", + } + }, + root=False, + attr_type=False, + ) + @pytest.mark.parametrize("invalid_char", ["\x00", "\x01", "\x0b", "\x1b", "\ufffe"]) @pytest.mark.parametrize("cdata", [False, True]) # @lat: [[tests#Conversion behavior#XML 1.0 forbidden characters are rejected]] From bf27193e3472092164458404f906890b9e269724 Mon Sep 17 00:00:00 2001 From: Vinit Kumar Date: Sun, 19 Jul 2026 20:24:32 +0530 Subject: [PATCH 10/10] fix: replace ambiguous XML character regex --- json2xml/dicttoxml.py | 30 ++++++++++++++++-------------- lat.md/tests.md | 2 +- tests/test_dict2xml.py | 29 ++++++++++++++++++++++++++++- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/json2xml/dicttoxml.py b/json2xml/dicttoxml.py index 14bfb65..cbc8c9b 100644 --- a/json2xml/dicttoxml.py +++ b/json2xml/dicttoxml.py @@ -2,7 +2,6 @@ import datetime import numbers -import re from collections.abc import Callable, Sequence from dataclasses import dataclass from decimal import Decimal @@ -18,14 +17,6 @@ _SAFE_RANDOM = SystemRandom() _XML_ESCAPE_CHARS = frozenset("&\"'<>") -# XML 1.0 character production: https://www.w3.org/TR/xml/#charsets -# XML 1.0 permits tab (U+0009), line feed (U+000A), carriage return -# (U+000D), and U+0020 onward, except for surrogate code points and the -# U+FFFE/U+FFFF noncharacters. This pattern matches only the forbidden ranges: -# disallowed C0 controls, UTF-16 surrogates, and the two BMP noncharacters. -_XML_INVALID_CHAR_RE = re.compile( - "[\x00-\x08\x0b\x0c\x0e-\x1f\ud800-\udfff\ufffe\uffff]" -) class _XMLWriter: @@ -137,12 +128,23 @@ def get_xml_type(val: Any) -> str: # @lat: [[behavior#XML output safety]] def _validate_xml_chars(value: str) -> None: """Reject characters excluded from the XML 1.0 Char production.""" - invalid_match = _XML_INVALID_CHAR_RE.search(value) - if invalid_match is not None: - codepoint = ord(invalid_match.group()) - raise ValueError( - f"Character U+{codepoint:04X} is not allowed in XML 1.0" + # XML 1.0 character production: https://www.w3.org/TR/xml/#charsets + # isprintable() keeps the common path in C; only strings containing a + # non-printable character need the explicit code-point boundary checks. + if not value or value.isprintable(): + return + + for character in value: + codepoint = ord(character) + is_forbidden_control = ( + codepoint < 0x20 and codepoint not in (0x09, 0x0A, 0x0D) ) + is_surrogate = 0xD800 <= codepoint <= 0xDFFF + is_bmp_noncharacter = codepoint in (0xFFFE, 0xFFFF) + if is_forbidden_control or is_surrogate or is_bmp_noncharacter: + raise ValueError( + f"Character U+{codepoint:04X} is not allowed in XML 1.0" + ) def escape_xml(s: str | int | float | numbers.Number | None) -> str: diff --git a/lat.md/tests.md b/lat.md/tests.md index 1a3527f..5022bc6 100644 --- a/lat.md/tests.md +++ b/lat.md/tests.md @@ -184,7 +184,7 @@ Caller-provided `type` attribute values containing XML 1.0-forbidden characters ### XML 1.0 forbidden characters are rejected -Text and CDATA output must reject forbidden XML 1.0 control characters before raw bytes are returned, while preserving valid tab, newline, and carriage-return characters. +Text and CDATA output must reject every forbidden XML 1.0 boundary before raw bytes are returned, while preserving valid whitespace and non-printable characters outside those ranges. ## XML helper behavior diff --git a/tests/test_dict2xml.py b/tests/test_dict2xml.py index c6927c5..3f42903 100644 --- a/tests/test_dict2xml.py +++ b/tests/test_dict2xml.py @@ -630,7 +630,21 @@ def test_custom_type_attribute_forbidden_chars_are_rejected(self) -> None: attr_type=False, ) - @pytest.mark.parametrize("invalid_char", ["\x00", "\x01", "\x0b", "\x1b", "\ufffe"]) + @pytest.mark.parametrize( + "invalid_char", + [ + "\x00", + "\x08", + "\x0b", + "\x0c", + "\x0e", + "\x1f", + "\ud800", + "\udfff", + "\ufffe", + "\uffff", + ], + ) @pytest.mark.parametrize("cdata", [False, True]) # @lat: [[tests#Conversion behavior#XML 1.0 forbidden characters are rejected]] def test_dicttoxml_rejects_xml_1_0_forbidden_characters( @@ -653,6 +667,19 @@ def test_dicttoxml_preserves_xml_1_0_whitespace_characters(self) -> None: assert result == b"tab\tline\nreturn\r" + @pytest.mark.parametrize("valid_char", ["\x7f", "\x85"]) + def test_dicttoxml_preserves_other_valid_nonprintable_characters( + self, valid_char: str + ) -> None: + """Non-printable characters outside the forbidden ranges remain valid.""" + result = dicttoxml.dicttoxml( + {"key": f"before{valid_char}after"}, + root=False, + attr_type=False, + ) + + assert result == f"before{valid_char}after".encode() + def test_dicttoxml_with_xml_namespaces(self) -> None: """Test dicttoxml with XML namespaces.""" data = {"key": "value"}