fix(ha): normalize.py round-trips custom HA !tag nodes

yaml.safe_load has no constructor for HA's `!include`/`!secret`/etc. tags
and raised ConstructorError on any real configuration.yaml. Add a
multi_constructor on `!` that generically wraps scalar/sequence/mapping
payloads in a TaggedValue, and a matching representer that re-emits the
same tag + payload — normalization stays idempotent across passes and key
sorting is unaffected by tagged values.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

scripts/ha/lib/normalize.py

@@ -4,6 +4,14 @@
 One function, one style, used everywhere data lands in the repo (see
 services/home-assistant/DESIGN.md, "Canonical format"): block collections,
 indent width 2, unlimited line width, sorted keys, UTF-8.
+
+HA configs use custom `!`-prefixed tags (`!include`, `!secret`, `!env_var`,
+`!input`, the `!include_dir_*` family, and any future addition) that
+`yaml.safe_load` has no constructor for and rejects with a ConstructorError.
+These tags are opaque to us — we never need to resolve `!include`, we just
+need to carry it through a parse/dump round-trip unchanged. `TaggedValue`
+plus the loader/dumper hooks below handle any `!xxx` tag generically
+(scalar, sequence, or mapping payload) without enumerating known tag names.
 """
 import json
 import sys
@@ -11,6 +19,50 @@ import sys
 import yaml
 
 
+class TaggedValue:
+    """Wraps a value parsed from a custom `!tag` node (see module docstring).
+
+    `tag` is the full tag text (e.g. `"!include"`); `value` is the plain
+    Python object (str/list/dict) constructed from the node's payload.
+    """
+
+    def __init__(self, tag, value):
+        self.tag = tag
+        self.value = value
+
+    def __eq__(self, other):
+        return (
+            isinstance(other, TaggedValue)
+            and self.tag == other.tag
+            and self.value == other.value
+        )
+
+    def __repr__(self):
+        return f"TaggedValue({self.tag!r}, {self.value!r})"
+
+
+class _CanonicalLoader(yaml.SafeLoader):
+    pass
+
+
+def _construct_tagged_value(loader, tag_suffix, node):
+    tag = "!" + tag_suffix
+    if isinstance(node, yaml.ScalarNode):
+        value = loader.construct_scalar(node)
+    elif isinstance(node, yaml.SequenceNode):
+        value = loader.construct_sequence(node, deep=True)
+    elif isinstance(node, yaml.MappingNode):
+        value = loader.construct_mapping(node, deep=True)
+    else:
+        raise yaml.constructor.ConstructorError(
+            None, None, f"unsupported node type for tag {tag!r}", node.start_mark
+        )
+    return TaggedValue(tag, value)
+
+
+_CanonicalLoader.add_multi_constructor("!", _construct_tagged_value)
+
+
 class _CanonicalDumper(yaml.SafeDumper):
     pass
 
@@ -19,7 +71,16 @@ def _represent_none(dumper, _value):
     return dumper.represent_scalar("tag:yaml.org,2002:null", "")
 
 
+def _represent_tagged_value(dumper, data):
+    if isinstance(data.value, dict):
+        return dumper.represent_mapping(data.tag, data.value)
+    if isinstance(data.value, list):
+        return dumper.represent_sequence(data.tag, data.value)
+    return dumper.represent_scalar(data.tag, data.value)
+
+
 _CanonicalDumper.add_representer(type(None), _represent_none)
+_CanonicalDumper.add_representer(TaggedValue, _represent_tagged_value)
 
 
 def dump_canonical_yaml(data):
@@ -35,9 +96,18 @@ def dump_canonical_yaml(data):
     )
 
 
+def parse_yaml_text(text):
+    """Parse YAML text, resolving HA's custom `!include` / `!secret` / etc.
+
+    tags to `TaggedValue` instances instead of raising ConstructorError
+    (what plain `yaml.safe_load` does).
+    """
+    return yaml.load(text, Loader=_CanonicalLoader)
+
+
 def normalize_yaml_text(text):
     """Parse YAML text then re-dump it through the canonical formatter."""
-    data = yaml.safe_load(text)
+    data = parse_yaml_text(text)
     return dump_canonical_yaml(data)
 
 

scripts/ha/tests/fixtures/configuration.yaml

@@ -0,0 +1,29 @@
+homeassistant:
+  name: Home
+  time_zone: Europe/Warsaw
+
+http:
+  server_port: 8123
+
+logger:
+  default: info
+
+automation: !include automations.yaml
+scene: !include scenes.yaml
+group: !include_dir_merge_named groups/
+
+mqtt:
+  broker: 192.168.1.10
+  username: !secret mqtt_username
+  password: !secret mqtt_password
+
+# Synthetic cases below are not real HA tags — they exist only to exercise
+# generic mapping/sequence node handling for custom `!tag`s (normalize.py
+# must not hardcode a list of known tag names).
+zone_test: !fake_mapping_tag
+  latitude: 52.1
+  longitude: 21.0
+
+platform_test: !fake_sequence_tag
+  - alpha
+  - beta

scripts/ha/tests/test_normalize_tags.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# HA custom `!tag` round-trip test for scripts/ha/lib/normalize.py.
+#
+# yaml.safe_load has no constructor for HA's `!include` / `!secret` / etc.
+# tags and raises ConstructorError on them. normalize.py must parse them
+# generically (scalar, sequence, and mapping payloads) and re-dump the same
+# tag + payload, with normalization staying deterministic across repeated
+# passes. Quoting style of tagged scalars is allowed to change (normalize.py
+# reformats the whole document already); what must not change is the tag
+# itself and the parsed value it carries.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+HA_LIB_DIR="$SCRIPT_DIR/../lib"
+FIXTURE="$SCRIPT_DIR/fixtures/configuration.yaml"
+
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT
+
+fail=0
+
+if ! python3 "$HA_LIB_DIR/normalize.py" yaml "$FIXTURE" > "$TMP/pass1.yaml" 2> "$TMP/pass1.err"; then
+  echo "FAIL: normalize.py raised on a configuration.yaml with custom HA tags" >&2
+  cat "$TMP/pass1.err" >&2
+  fail=1
+else
+  echo "PASS: normalize.py parses custom HA tags without ConstructorError"
+fi
+
+expected_tags=(
+  "!include"
+  "!include_dir_merge_named"
+  "!secret"
+  "!fake_mapping_tag"
+  "!fake_sequence_tag"
+)
+for tag in "${expected_tags[@]}"; do
+  if ! grep -qF "$tag" "$TMP/pass1.yaml"; then
+    echo "FAIL: expected tag '$tag' missing from normalized output" >&2
+    fail=1
+  fi
+done
+if [[ "$fail" -eq 0 ]]; then
+  echo "PASS: all custom tags present in normalized output"
+fi
+
+# Structural round-trip: parsing the normalized output back through the same
+# loader must yield the exact same tagged values (scalar/sequence/mapping
+# payloads) as parsing the original fixture — normalization must not lose or
+# mutate data carried under a custom tag.
+if ! python3 - "$HA_LIB_DIR" "$FIXTURE" "$TMP/pass1.yaml" <<'PYEOF'
+import sys
+
+lib_dir, fixture_path, normalized_path = sys.argv[1:4]
+sys.path.insert(0, lib_dir)
+from normalize import parse_yaml_text
+
+with open(fixture_path, encoding="utf-8") as f:
+    original = parse_yaml_text(f.read())
+with open(normalized_path, encoding="utf-8") as f:
+    normalized = parse_yaml_text(f.read())
+
+if original != normalized:
+    print(f"structural mismatch:\noriginal:  {original!r}\nnormalized: {normalized!r}", file=sys.stderr)
+    sys.exit(1)
+PYEOF
+then
+  echo "FAIL: normalized output does not structurally round-trip to the original tagged values" >&2
+  fail=1
+else
+  echo "PASS: normalized output structurally round-trips (tags + payloads unchanged)"
+fi
+
+# Idempotency: normalizing the already-normalized output a second time must
+# produce byte-identical text (tags preserved through N passes).
+if ! python3 "$HA_LIB_DIR/normalize.py" yaml "$TMP/pass1.yaml" > "$TMP/pass2.yaml" 2> "$TMP/pass2.err"; then
+  echo "FAIL: second normalization pass raised on its own output" >&2
+  cat "$TMP/pass2.err" >&2
+  fail=1
+elif ! diff "$TMP/pass1.yaml" "$TMP/pass2.yaml" > /dev/null; then
+  echo "FAIL: normalize.py is not idempotent across two passes" >&2
+  diff "$TMP/pass1.yaml" "$TMP/pass2.yaml" >&2 || true
+  fail=1
+else
+  echo "PASS: normalize.py output is byte-identical across two passes"
+fi
+
+exit "$fail"