From 1ce8383f2ea0eb2f8d4c461fac94cce184b6c04c Mon Sep 17 00:00:00 2001 From: oskar Date: Tue, 21 Jul 2026 17:41:12 +0200 Subject: [PATCH] fix(ha): normalize.py round-trips custom HA `!tag` nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit yaml.safe_load has no constructor for HA's `!include`/`!secret`/etc. tags and raised ConstructorError on any real configuration.yaml. Add a multi_constructor on `!` that generically wraps scalar/sequence/mapping payloads in a TaggedValue, and a matching representer that re-emits the same tag + payload — normalization stays idempotent across passes and key sorting is unaffected by tagged values. Co-Authored-By: Claude Sonnet 5 --- scripts/ha/lib/normalize.py | 72 +++++++++++++++- scripts/ha/tests/fixtures/configuration.yaml | 29 +++++++ scripts/ha/tests/test_normalize_tags.sh | 88 ++++++++++++++++++++ 3 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 scripts/ha/tests/fixtures/configuration.yaml create mode 100755 scripts/ha/tests/test_normalize_tags.sh diff --git a/scripts/ha/lib/normalize.py b/scripts/ha/lib/normalize.py index a8750e0..711e07d 100755 --- a/scripts/ha/lib/normalize.py +++ b/scripts/ha/lib/normalize.py @@ -4,6 +4,14 @@ One function, one style, used everywhere data lands in the repo (see services/home-assistant/DESIGN.md, "Canonical format"): block collections, indent width 2, unlimited line width, sorted keys, UTF-8. + +HA configs use custom `!`-prefixed tags (`!include`, `!secret`, `!env_var`, +`!input`, the `!include_dir_*` family, and any future addition) that +`yaml.safe_load` has no constructor for and rejects with a ConstructorError. +These tags are opaque to us — we never need to resolve `!include`, we just +need to carry it through a parse/dump round-trip unchanged. `TaggedValue` +plus the loader/dumper hooks below handle any `!xxx` tag generically +(scalar, sequence, or mapping payload) without enumerating known tag names. """ import json import sys @@ -11,6 +19,50 @@ import sys import yaml +class TaggedValue: + """Wraps a value parsed from a custom `!tag` node (see module docstring). + + `tag` is the full tag text (e.g. `"!include"`); `value` is the plain + Python object (str/list/dict) constructed from the node's payload. + """ + + def __init__(self, tag, value): + self.tag = tag + self.value = value + + def __eq__(self, other): + return ( + isinstance(other, TaggedValue) + and self.tag == other.tag + and self.value == other.value + ) + + def __repr__(self): + return f"TaggedValue({self.tag!r}, {self.value!r})" + + +class _CanonicalLoader(yaml.SafeLoader): + pass + + +def _construct_tagged_value(loader, tag_suffix, node): + tag = "!" + tag_suffix + if isinstance(node, yaml.ScalarNode): + value = loader.construct_scalar(node) + elif isinstance(node, yaml.SequenceNode): + value = loader.construct_sequence(node, deep=True) + elif isinstance(node, yaml.MappingNode): + value = loader.construct_mapping(node, deep=True) + else: + raise yaml.constructor.ConstructorError( + None, None, f"unsupported node type for tag {tag!r}", node.start_mark + ) + return TaggedValue(tag, value) + + +_CanonicalLoader.add_multi_constructor("!", _construct_tagged_value) + + class _CanonicalDumper(yaml.SafeDumper): pass @@ -19,7 +71,16 @@ def _represent_none(dumper, _value): return dumper.represent_scalar("tag:yaml.org,2002:null", "") +def _represent_tagged_value(dumper, data): + if isinstance(data.value, dict): + return dumper.represent_mapping(data.tag, data.value) + if isinstance(data.value, list): + return dumper.represent_sequence(data.tag, data.value) + return dumper.represent_scalar(data.tag, data.value) + + _CanonicalDumper.add_representer(type(None), _represent_none) +_CanonicalDumper.add_representer(TaggedValue, _represent_tagged_value) def dump_canonical_yaml(data): @@ -35,9 +96,18 @@ def dump_canonical_yaml(data): ) +def parse_yaml_text(text): + """Parse YAML text, resolving HA's custom `!include` / `!secret` / etc. + + tags to `TaggedValue` instances instead of raising ConstructorError + (what plain `yaml.safe_load` does). + """ + return yaml.load(text, Loader=_CanonicalLoader) + + def normalize_yaml_text(text): """Parse YAML text then re-dump it through the canonical formatter.""" - data = yaml.safe_load(text) + data = parse_yaml_text(text) return dump_canonical_yaml(data) diff --git a/scripts/ha/tests/fixtures/configuration.yaml b/scripts/ha/tests/fixtures/configuration.yaml new file mode 100644 index 0000000..8062787 --- /dev/null +++ b/scripts/ha/tests/fixtures/configuration.yaml @@ -0,0 +1,29 @@ +homeassistant: + name: Home + time_zone: Europe/Warsaw + +http: + server_port: 8123 + +logger: + default: info + +automation: !include automations.yaml +scene: !include scenes.yaml +group: !include_dir_merge_named groups/ + +mqtt: + broker: 192.168.1.10 + username: !secret mqtt_username + password: !secret mqtt_password + +# Synthetic cases below are not real HA tags — they exist only to exercise +# generic mapping/sequence node handling for custom `!tag`s (normalize.py +# must not hardcode a list of known tag names). +zone_test: !fake_mapping_tag + latitude: 52.1 + longitude: 21.0 + +platform_test: !fake_sequence_tag + - alpha + - beta diff --git a/scripts/ha/tests/test_normalize_tags.sh b/scripts/ha/tests/test_normalize_tags.sh new file mode 100755 index 0000000..c4eaffd --- /dev/null +++ b/scripts/ha/tests/test_normalize_tags.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# HA custom `!tag` round-trip test for scripts/ha/lib/normalize.py. +# +# yaml.safe_load has no constructor for HA's `!include` / `!secret` / etc. +# tags and raises ConstructorError on them. normalize.py must parse them +# generically (scalar, sequence, and mapping payloads) and re-dump the same +# tag + payload, with normalization staying deterministic across repeated +# passes. Quoting style of tagged scalars is allowed to change (normalize.py +# reformats the whole document already); what must not change is the tag +# itself and the parsed value it carries. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +HA_LIB_DIR="$SCRIPT_DIR/../lib" +FIXTURE="$SCRIPT_DIR/fixtures/configuration.yaml" + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +fail=0 + +if ! python3 "$HA_LIB_DIR/normalize.py" yaml "$FIXTURE" > "$TMP/pass1.yaml" 2> "$TMP/pass1.err"; then + echo "FAIL: normalize.py raised on a configuration.yaml with custom HA tags" >&2 + cat "$TMP/pass1.err" >&2 + fail=1 +else + echo "PASS: normalize.py parses custom HA tags without ConstructorError" +fi + +expected_tags=( + "!include" + "!include_dir_merge_named" + "!secret" + "!fake_mapping_tag" + "!fake_sequence_tag" +) +for tag in "${expected_tags[@]}"; do + if ! grep -qF "$tag" "$TMP/pass1.yaml"; then + echo "FAIL: expected tag '$tag' missing from normalized output" >&2 + fail=1 + fi +done +if [[ "$fail" -eq 0 ]]; then + echo "PASS: all custom tags present in normalized output" +fi + +# Structural round-trip: parsing the normalized output back through the same +# loader must yield the exact same tagged values (scalar/sequence/mapping +# payloads) as parsing the original fixture — normalization must not lose or +# mutate data carried under a custom tag. +if ! python3 - "$HA_LIB_DIR" "$FIXTURE" "$TMP/pass1.yaml" <<'PYEOF' +import sys + +lib_dir, fixture_path, normalized_path = sys.argv[1:4] +sys.path.insert(0, lib_dir) +from normalize import parse_yaml_text + +with open(fixture_path, encoding="utf-8") as f: + original = parse_yaml_text(f.read()) +with open(normalized_path, encoding="utf-8") as f: + normalized = parse_yaml_text(f.read()) + +if original != normalized: + print(f"structural mismatch:\noriginal: {original!r}\nnormalized: {normalized!r}", file=sys.stderr) + sys.exit(1) +PYEOF +then + echo "FAIL: normalized output does not structurally round-trip to the original tagged values" >&2 + fail=1 +else + echo "PASS: normalized output structurally round-trips (tags + payloads unchanged)" +fi + +# Idempotency: normalizing the already-normalized output a second time must +# produce byte-identical text (tags preserved through N passes). +if ! python3 "$HA_LIB_DIR/normalize.py" yaml "$TMP/pass1.yaml" > "$TMP/pass2.yaml" 2> "$TMP/pass2.err"; then + echo "FAIL: second normalization pass raised on its own output" >&2 + cat "$TMP/pass2.err" >&2 + fail=1 +elif ! diff "$TMP/pass1.yaml" "$TMP/pass2.yaml" > /dev/null; then + echo "FAIL: normalize.py is not idempotent across two passes" >&2 + diff "$TMP/pass1.yaml" "$TMP/pass2.yaml" >&2 || true + fail=1 +else + echo "PASS: normalize.py output is byte-identical across two passes" +fi + +exit "$fail"