From ab6ad27cb2b66956443ff8591b69fa9a97276399 Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Tue, 14 Jul 2026 18:54:00 +0300 Subject: [PATCH 1/2] fix: accept Compose's 'host=ip' separator in extra_hosts Compose documents extra_hosts list entries as `host=ip` and also accepts the legacy `host:ip`. compose2pod only ever split on ':', so the documented form was accepted by the gate and then silently mangled: extra_hosts: ["somehost=162.242.195.82"] -> podman pod create ... --add-host "somehost=162.242.195.82:" The whole string became the hostname and the address came out empty. An entry with no separator at all had the same face: accepted, emitted as `--add-host "no-separator:"`. keys.split_extra_host is now the one reader every site shares -- the gate, the emitter, and the extends merge -- so they cannot disagree about where an entry divides. '=' wins when present, because an IPv6 address is full of colons and splitting on the first one would tear it apart; the colon form still splits on the first colon only, so `myhost:::1` keeps working. An entry with neither separator now raises. --- architecture/supported-subset.md | 18 ++-- compose2pod/extends.py | 15 ++-- compose2pod/keys.py | 18 ++++ compose2pod/pod.py | 20 ++++- ...6-07-14.06-extra-hosts-equals-separator.md | 85 +++++++++++++++++++ tests/test_extends.py | 27 +++++- tests/test_pod.py | 30 +++++++ 7 files changed, 196 insertions(+), 17 deletions(-) create mode 100644 planning/changes/2026-07-14.06-extra-hosts-equals-separator.md diff --git a/architecture/supported-subset.md b/architecture/supported-subset.md index 3647c1a..2699d7d 100644 --- a/architecture/supported-subset.md +++ b/architecture/supported-subset.md @@ -252,9 +252,9 @@ slot (`architecture/glossary.md`). forms are declared in `extends.py` (`_STRUCTURAL_*`, `_SCALAR_FORM_KEYS`) and kept honest by a test asserting, for every mergeable key, that a form the gate refuses standalone is refused through `extends` too. - `extra_hosts`' list form is `host:ip` — colon-separated, split on the - *first* colon so an IPv6 address survives (`myhost:::1` → - `{myhost: ::1}`). + `extra_hosts`' list form divides on `=` or `:` (`keys.split_extra_host`, + the one reader the gate, the emitter and the merge all share), so an IPv6 + address survives either way. - **Refused loudly:** cross-file `extends: {file: ..., service: ...}`; a bare-string (or any other non-mapping) `extends`; an unrecognized key under `extends` other than `service`; a non-string `service`; a `service` @@ -287,8 +287,16 @@ flags. compose2pod hoists them onto `podman pod create` instead - **Value shapes:** `dns`/`dns_search`/`dns_opt` accept a string or list of strings; `sysctls` accepts a mapping (`key: value`) or a list of `"key=value"` strings, each value a string or number; `extra_hosts` - accepts a list (`- host:ip`) or mapping (`host: ip`) — an IPv6 address - value keeps its colons, unlike a plain host:ip pair. A `${VAR}` inside a + accepts a list or a mapping (`host: ip`). A list entry divides on `=` + (Compose's documented separator, `- somehost=162.242.195.82`) or on the + legacy `:` (`- somehost:162.242.195.82`) — `=` wins when both appear, + because an IPv6 address is itself full of colons and splitting on the first + one would tear it apart (`myhostv6=::1`); the colon form splits on the + *first* colon only, so `myhost:::1` works too. Every reader — the gate, the + emitter, and the `extends` merge — goes through the one shared + `keys.split_extra_host`, so they cannot disagree about where an entry + divides. An entry with neither separator has no address and raises, rather + than emitting a malformed `--add-host`. A `${VAR}` inside a value stays live at run time (see Variable interpolation, below) and counts toward `referenced_variables`. - **Aggregation is closure-scoped:** computed over the target's dependency diff --git a/compose2pod/extends.py b/compose2pod/extends.py index 986f470..8290d3c 100644 --- a/compose2pod/extends.py +++ b/compose2pod/extends.py @@ -3,7 +3,7 @@ from typing import Any from compose2pod.exceptions import UnsupportedComposeError -from compose2pod.keys import SERVICE_KEYS, as_list, pairs_to_mapping +from compose2pod.keys import SERVICE_KEYS, as_list, pairs_to_mapping, split_extra_host # Merge policy for keys with a SERVICE_KEYS KeySpec comes from spec.merge (see @@ -49,16 +49,17 @@ def _extends_target(name: str, ext: Any) -> str: # noqa: ANN401 - Compose value def _extra_hosts_to_mapping(name: str, value: list[Any]) -> dict[str, Any]: """Normalize list-form `extra_hosts` ('host:ip') to a mapping. - Separated by a colon, not '=', so `pairs_to_mapping` would mangle the whole - entry into a single `{'host:ip': None}` key. Split on the *first* colon only: - an IPv6 address is itself full of them (`myhost:::1` -> `{'myhost': '::1'}`). + Not '='-or-'='-only: `pairs_to_mapping` would mangle the whole entry into a + single `{'host:ip': None}` key. `keys.split_extra_host` is the one shared + reader -- Compose's documented `host=ip` and the legacy `host:ip` both work, + and an IPv6 address keeps its colons. """ result: dict[str, Any] = {} for item in value: - if not isinstance(item, str) or ":" not in item: - msg = f"service {name!r}: extra_hosts entries must be 'host:ip' strings" + if not isinstance(item, str) or ("=" not in item and ":" not in item): + msg = f"service {name!r}: extra_hosts entries must be 'host=ip' or 'host:ip' strings" raise UnsupportedComposeError(msg) - host, _sep, address = item.partition(":") + host, address = split_extra_host(item) result[host] = address return result diff --git a/compose2pod/keys.py b/compose2pod/keys.py index c03617f..97f9bda 100644 --- a/compose2pod/keys.py +++ b/compose2pod/keys.py @@ -225,6 +225,24 @@ def emit(value: Any) -> list[Token]: # noqa: ANN401 - Compose values are untype return KeySpec(validate=validate_map, emit=emit, merge=_merge_map) +def split_extra_host(entry: str) -> tuple[str, str]: + """Split an `extra_hosts` entry into (host, address). + + Compose documents the separator as '=' ('somehost=162.242.195.82') and also + accepts the legacy 'host:ip'. '=' wins when present, because an IPv6 address + is itself full of colons and splitting on the first one would tear it apart: + 'myhostv6=::1' -> ('myhostv6', '::1'). The colon form splits on the *first* + colon only, so 'myhost:::1' -> ('myhost', '::1') still works. + + The one shared reader for every site that parses an entry -- the gate, the + emitter, and the `extends` merge -- so they cannot disagree about where an + entry divides. + """ + separator = "=" if "=" in entry else ":" + host, _sep, address = entry.partition(separator) + return host, address + + def extra_host_pairs(value: list[Any] | dict[str, Any]) -> list[Any]: """Compose extra_hosts as 'host:ip' entries; map values keep their colons (IPv6-safe).""" if isinstance(value, list): diff --git a/compose2pod/pod.py b/compose2pod/pod.py index 1c6b336..3bb8375 100644 --- a/compose2pod/pod.py +++ b/compose2pod/pod.py @@ -3,7 +3,7 @@ from typing import Any from compose2pod.exceptions import UnsupportedComposeError -from compose2pod.keys import Expand, Token, extra_host_pairs, validate_map +from compose2pod.keys import Expand, Token, extra_host_pairs, split_extra_host, validate_map _DNS_KEYS = {"dns": "--dns", "dns_search": "--dns-search", "dns_opt": "--dns-option"} @@ -40,6 +40,21 @@ def _sysctl_pairs(name: str, value: Any) -> list[tuple[str, str]]: # noqa: ANN4 raise UnsupportedComposeError(msg) +def _check_extra_host_separators(name: str, value: Any) -> None: # noqa: ANN401 - Compose values are untyped + """Check each list-form entry actually divides into a host and an address. + + An entry with neither separator has no address to emit, and would render as + a malformed `--add-host "no-separator:"`. The mapping form cannot have this + problem: its key and value are already separate. + """ + if not isinstance(value, list): + return + for entry in value: + if "=" not in entry and ":" not in entry: + msg = f"service {name!r}: extra_hosts entries must be 'host=ip' or 'host:ip' (got {entry!r})" + raise UnsupportedComposeError(msg) + + def validate_pod_options(name: str, svc: dict[str, Any]) -> None: """Shape-check a service's pod-level dns/sysctls declarations.""" for key in _DNS_KEYS: @@ -49,6 +64,7 @@ def validate_pod_options(name: str, svc: dict[str, Any]) -> None: _sysctl_pairs(name, svc["sysctls"]) if "extra_hosts" in svc: validate_map(name, "extra_hosts", svc["extra_hosts"]) + _check_extra_host_separators(name, svc["extra_hosts"]) def uses_pod_options(services: dict[str, Any]) -> bool: @@ -105,7 +121,7 @@ def _add_host_flags(services: dict[str, Any], order: list[str], hosts: list[str] if "extra_hosts" not in svc: continue for entry in extra_host_pairs(svc["extra_hosts"]): - host, _sep, addr = str(entry).partition(":") + host, addr = split_extra_host(str(entry)) if merged.get(host, addr) != addr: msg = f"service {name!r}: conflicting host {host!r} ({merged[host]!r} vs {addr!r})" raise UnsupportedComposeError(msg) diff --git a/planning/changes/2026-07-14.06-extra-hosts-equals-separator.md b/planning/changes/2026-07-14.06-extra-hosts-equals-separator.md new file mode 100644 index 0000000..af89187 --- /dev/null +++ b/planning/changes/2026-07-14.06-extra-hosts-equals-separator.md @@ -0,0 +1,85 @@ +--- +summary: extra_hosts list entries accept Compose's documented 'host=ip' separator as well as the legacy 'host:ip', and an entry with neither is refused instead of emitting a malformed --add-host. +--- + +# Design: Support the `host=ip` separator in extra_hosts + +## Summary + +Compose documents `extra_hosts` list entries as `host=ip`, and also accepts the +legacy `host:ip`. compose2pod only ever split on `:` — so the documented form +was accepted by the gate and then **silently mangled** into a malformed +`--add-host`. One shared splitter now handles both, and an entry with neither +separator is refused rather than emitted as garbage. + +## Motivation + +Silent corruption, exit code 0: + +```yaml +extra_hosts: + - "somehost=162.242.195.82" # Compose's own documented form +``` +``` +podman pod create ... --add-host "somehost=162.242.195.82:" +``` + +The whole string became the *hostname* and the address came out empty. +`pod._add_host_flags` did `entry.partition(":")`, which on a `=`-separated entry +finds no colon and yields `(entry, "", "")`. + +The same shape has a second face: an entry with **no separator at all** +(`- "no-colon"`) is likewise accepted and emits `--add-host "no-colon:"`. The +gate checks `extra_hosts` is a list of strings but never that an entry is +actually a `host`/`address` pair. + +This is the silent-corruption class the structural-key gate work +(`2026-07-13.10`) set out to eliminate, in a corner it did not reach. + +## Design + +One splitter, in `keys.py` beside the other cross-module primitives, used by +every site that reads an entry: + +```python +def split_extra_host(entry: str) -> tuple[str, str]: + if "=" in entry: # Compose's documented separator + host, _sep, address = entry.partition("=") + else: # legacy 'host:ip' + host, _sep, address = entry.partition(":") + return host, address +``` + +`=` wins when present, because an **IPv6 address is full of colons** and would +otherwise be split apart: `myhostv6=::1` → `('myhostv6', '::1')`. The colon form +still splits on the *first* colon only, so `myhost:::1` → `('myhost', '::1')` +keeps working. + +Three readers share it — `pod._add_host_flags` (emit), `extends`' +list-to-mapping normalizer (merge), and a new gate check that an entry contains +a separator at all. Previously each parsed the entry itself; a single primitive +is what stops them drifting, the same reason `keys.py` already owns +`extra_host_pairs`. + +## Non-goals + +- Not changing the **mapping** form (`host: ip`), which is unambiguous and + already correct. +- Not validating that the address is a well-formed IP. Podman rejects a bad + address at run time, and compose2pod does not otherwise parse addresses. + +## Testing + +`just test-ci` at 100%. Cases: `=` form (v4 and v6), `:` form (v4 and v6), +mapping form unchanged, an entry with no separator refused at the gate, the +`extends` merge path accepting both separators, and a conflict between two +services still refused. Plus an emit-level check that both separators produce +the same `--add-host` flag. + +## Risk + +- **A document relying on the mangled output** — impossible: the old output was + a malformed host entry with an empty address, which podman could not have + resolved. Nothing can depend on it. +- An entry with no separator now raises where it was silently accepted. That is + the intended correction; it emitted a broken flag. diff --git a/tests/test_extends.py b/tests/test_extends.py index 15d063a..4e85373 100644 --- a/tests/test_extends.py +++ b/tests/test_extends.py @@ -491,14 +491,35 @@ def test_ipv6_address_keeps_its_colons(self) -> None: } assert resolve_extends(doc)["services"]["web"]["extra_hosts"] == {"a": "1.1.1.1", "myhost": "::1"} - def test_entry_without_a_colon_is_refused(self) -> None: + def test_equals_separator_form_merges(self) -> None: + # Compose's documented separator, through the merge path. + doc = { + "services": { + "base": {"image": "x", "extra_hosts": {"a": "1.1.1.1"}}, + "web": {"extends": {"service": "base"}, "extra_hosts": ["somehost=162.242.195.82"]}, + } + } + merged = resolve_extends(doc)["services"]["web"]["extra_hosts"] + assert merged == {"a": "1.1.1.1", "somehost": "162.242.195.82"} + + def test_equals_separator_keeps_ipv6(self) -> None: + doc = { + "services": { + "base": {"image": "x", "extra_hosts": {"a": "1.1.1.1"}}, + "web": {"extends": {"service": "base"}, "extra_hosts": ["myhostv6=::1"]}, + } + } + merged = resolve_extends(doc)["services"]["web"]["extra_hosts"] + assert merged == {"a": "1.1.1.1", "myhostv6": "::1"} + + def test_entry_without_a_separator_is_refused(self) -> None: doc = { "services": { "base": {"image": "x", "extra_hosts": {"a": "1.1.1.1"}}, "web": {"extends": {"service": "base"}, "extra_hosts": ["no-colon-here"]}, } } - with pytest.raises(UnsupportedComposeError, match="extra_hosts entries must be 'host:ip' strings"): + with pytest.raises(UnsupportedComposeError, match="extra_hosts entries must be 'host=ip' or 'host:ip'"): resolve_extends(doc) def test_non_string_entry_is_refused(self) -> None: @@ -508,7 +529,7 @@ def test_non_string_entry_is_refused(self) -> None: "web": {"extends": {"service": "base"}, "extra_hosts": [5]}, } } - with pytest.raises(UnsupportedComposeError, match="extra_hosts entries must be 'host:ip' strings"): + with pytest.raises(UnsupportedComposeError, match="extra_hosts entries must be 'host=ip' or 'host:ip'"): resolve_extends(doc) diff --git a/tests/test_pod.py b/tests/test_pod.py index c44bb55..95fb4b4 100644 --- a/tests/test_pod.py +++ b/tests/test_pod.py @@ -163,3 +163,33 @@ def test_conflicting_alias_and_extra_hosts_refused(self) -> None: def test_non_closure_service_extra_hosts_ignored(self) -> None: services = {"a": {"image": "x"}, "extra": {"extra_hosts": {"db": "10.0.0.5"}}} assert pod_create_flags(services, ["a"], []) == [] + + +class TestExtraHostsSeparators: + """Compose documents 'host=ip'; the legacy 'host:ip' also works. Both must.""" + + def _flags(self, entries: object) -> list[str]: + services = {"a": {"image": "x", "extra_hosts": entries}} + return [t.value if isinstance(t, Expand) else t for t in pod_create_flags(services, ["a"], [])] + + def test_equals_separator_is_split_correctly(self) -> None: + # Used to emit the whole string as the hostname: 'somehost=1.2.3.4:' + assert self._flags(["somehost=162.242.195.82"]) == ["--add-host", "somehost:162.242.195.82"] + + def test_colon_separator_still_works(self) -> None: + assert self._flags(["somehost:162.242.195.82"]) == ["--add-host", "somehost:162.242.195.82"] + + def test_equals_separator_keeps_ipv6(self) -> None: + # '=' wins over ':' precisely because an IPv6 address is full of colons. + assert self._flags(["myhostv6=::1"]) == ["--add-host", "myhostv6:::1"] + + def test_colon_separator_keeps_ipv6(self) -> None: + assert self._flags(["myhost:::1"]) == ["--add-host", "myhost:::1"] + + def test_mapping_form_is_unchanged(self) -> None: + assert self._flags({"somehost": "162.242.195.82"}) == ["--add-host", "somehost:162.242.195.82"] + + def test_entry_with_no_separator_is_refused(self) -> None: + # Used to be accepted and emit the malformed '--add-host "no-separator:"' + with pytest.raises(UnsupportedComposeError, match="extra_hosts entries must be 'host=ip' or 'host:ip'"): + validate_pod_options("a", {"image": "x", "extra_hosts": ["no-separator"]}) From 6575909222a8b14743bd818b7b26c3c19a6c01a0 Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Tue, 14 Jul 2026 19:03:48 +0300 Subject: [PATCH 2/2] fix: read extra_hosts as pairs instead of joining and re-splitting The first pass re-split every entry -- including mapping-form ones -- through split_extra_host, after extra_host_pairs had joined them into 'host:address'. With '=' now winning over ':', a mapping value containing an '=' re-divided at the wrong character: extra_hosts: {somehost: "weird=address"} -> --add-host "somehost:weird:address" (host and address both wrong) Silent, exit 0 -- the same corruption class this change exists to close, reintroduced one layer up. The mapping form arrives already divided, so it should never be joined and re-parsed. keys.extra_host_entries yields (host, address) pairs from either form: the mapping straight through (via _render_scalar, so a boolean address still normalizes like docker compose config), the list via split_extra_host. extra_host_pairs had no other caller and is gone. --- compose2pod/extends.py | 4 ++-- compose2pod/keys.py | 19 ++++++++++++++----- compose2pod/parsing.py | 2 +- compose2pod/pod.py | 5 ++--- tests/test_pod.py | 5 +++++ 5 files changed, 24 insertions(+), 11 deletions(-) diff --git a/compose2pod/extends.py b/compose2pod/extends.py index 8290d3c..c4acced 100644 --- a/compose2pod/extends.py +++ b/compose2pod/extends.py @@ -49,8 +49,8 @@ def _extends_target(name: str, ext: Any) -> str: # noqa: ANN401 - Compose value def _extra_hosts_to_mapping(name: str, value: list[Any]) -> dict[str, Any]: """Normalize list-form `extra_hosts` ('host:ip') to a mapping. - Not '='-or-'='-only: `pairs_to_mapping` would mangle the whole entry into a - single `{'host:ip': None}` key. `keys.split_extra_host` is the one shared + `pairs_to_mapping` would mangle an entry into a single `{'host:ip': None}` + key, since it splits on '=' only. `keys.split_extra_host` is the one shared reader -- Compose's documented `host=ip` and the legacy `host:ip` both work, and an IPv6 address keeps its colons. """ diff --git a/compose2pod/keys.py b/compose2pod/keys.py index 97f9bda..59426a8 100644 --- a/compose2pod/keys.py +++ b/compose2pod/keys.py @@ -243,11 +243,20 @@ def split_extra_host(entry: str) -> tuple[str, str]: return host, address -def extra_host_pairs(value: list[Any] | dict[str, Any]) -> list[Any]: - """Compose extra_hosts as 'host:ip' entries; map values keep their colons (IPv6-safe).""" - if isinstance(value, list): - return value - return [f"{host}:{_render_scalar(ip)}" for host, ip in value.items()] +def extra_host_entries(value: list[Any] | dict[str, Any]) -> list[tuple[str, str]]: + """Compose extra_hosts as (host, address) pairs, from either form. + + The mapping form arrives already divided, so it is read straight through; + only the list form needs splitting. Joining a mapping into 'host:address' + and re-splitting it would be lossy -- an address containing '=' would then + re-divide at the wrong character. + """ + if isinstance(value, dict): + # `_render_scalar` so a boolean address normalizes like `docker compose + # config` ('true', not Python's 'True') -- the same rule every other map + # value follows. + return [(str(host), _render_scalar(address)) for host, address in value.items()] + return [split_extra_host(str(item)) for item in value] def _validate_pull_policy(name: str, key: str, value: Any) -> None: # noqa: ANN401 - Compose values are untyped diff --git a/compose2pod/parsing.py b/compose2pod/parsing.py index 2c1bbef..c8ef4b6 100644 --- a/compose2pod/parsing.py +++ b/compose2pod/parsing.py @@ -205,7 +205,7 @@ def _require_string_keys_deep(where: str, node: Any) -> None: # noqa: ANN401 - - Mapping-key consumers that don't crash but f-string-interpolate the key straight into a flag value, silently leaking the Python repr of a bool or int into the emitted script (`keys.key_value_pairs` -- environment/ - labels/annotations, `keys.extra_host_pairs`, `keys._ulimit_args`, + labels/annotations, `keys.extra_host_entries`, `keys._ulimit_args`, `pod._sysctl_pairs`). This class is corruption, not a crash, and is the one the old hand-placed `require_string_keys` call sites never covered -- they were only wired into mappings whose keys get sorted or diff --git a/compose2pod/pod.py b/compose2pod/pod.py index 3bb8375..fd0363d 100644 --- a/compose2pod/pod.py +++ b/compose2pod/pod.py @@ -3,7 +3,7 @@ from typing import Any from compose2pod.exceptions import UnsupportedComposeError -from compose2pod.keys import Expand, Token, extra_host_pairs, split_extra_host, validate_map +from compose2pod.keys import Expand, Token, extra_host_entries, validate_map _DNS_KEYS = {"dns": "--dns", "dns_search": "--dns-search", "dns_opt": "--dns-option"} @@ -120,8 +120,7 @@ def _add_host_flags(services: dict[str, Any], order: list[str], hosts: list[str] svc = services[name] if "extra_hosts" not in svc: continue - for entry in extra_host_pairs(svc["extra_hosts"]): - host, addr = split_extra_host(str(entry)) + for host, addr in extra_host_entries(svc["extra_hosts"]): if merged.get(host, addr) != addr: msg = f"service {name!r}: conflicting host {host!r} ({merged[host]!r} vs {addr!r})" raise UnsupportedComposeError(msg) diff --git a/tests/test_pod.py b/tests/test_pod.py index 95fb4b4..e12290b 100644 --- a/tests/test_pod.py +++ b/tests/test_pod.py @@ -193,3 +193,8 @@ def test_entry_with_no_separator_is_refused(self) -> None: # Used to be accepted and emit the malformed '--add-host "no-separator:"' with pytest.raises(UnsupportedComposeError, match="extra_hosts entries must be 'host=ip' or 'host:ip'"): validate_pod_options("a", {"image": "x", "extra_hosts": ["no-separator"]}) + + def test_mapping_value_containing_an_equals_is_not_re_split(self) -> None: + # The mapping form arrives already divided. Joining it into 'host:address' + # and re-splitting would divide at the '=' instead. + assert self._flags({"somehost": "weird=address"}) == ["--add-host", "somehost:weird=address"]