From 81bf79c45489b3fcdcdcd8cb6bc051ff24e8946a Mon Sep 17 00:00:00 2001 From: ciltocruz Date: Mon, 19 Oct 2026 11:46:15 +0200 Subject: [PATCH] fix: re-attach or drop stale monitor when a container is recreated (404) When a container is recreated (Watchtower, docker-compose up -d, ...), the monitor keeps a handle for the old container id, so every poll fails with 'DockerError 404' forever and retries every 60s. If the destroy event is missed (docker proxy, races), a later create event bails out with 'Container already monitored' without replacing the stale monitor. - On 404, try to re-attach to a container with the same name; on success the monitor continues with the fresh handle. - If the container does not exist anymore (2 consecutive 404s), ask the DockerAPI instance to drop this monitor so a later create event can re-add it. - _container_add now replaces a stale monitor instead of returning early. Fixes #188 --- custom_components/monitor_docker/helpers.py | 99 ++++++++++++++++++--- 1 file changed, 89 insertions(+), 10 deletions(-) diff --git a/custom_components/monitor_docker/helpers.py b/custom_components/monitor_docker/helpers.py index 95bb689..b1d9f8d 100644 --- a/custom_components/monitor_docker/helpers.py +++ b/custom_components/monitor_docker/helpers.py @@ -269,6 +269,7 @@ async def init(self, startCount=0): self._config, self._api, cname, + parent=self, ) await self._containers[cname].init() @@ -615,16 +616,39 @@ async def _container_create_destroy(self) -> None: ) ############################################################# + async def on_container_gone(self, cname: str) -> None: + """Drop the monitor of a container that no longer exists. + + Called from a container monitor when the Docker API keeps answering + 404 (e.g. the container was recreated with a new id and the destroy + event was missed). Removing the monitor here allows a later create + event to re-add it with a fresh handle. + """ + if cname in self._containers: + _LOGGER.info( + "[%s] %s: Container does not exist anymore, removing monitor", + self._instance, + cname, + ) + asyncio.create_task(self._container_remove(cname)) + async def _container_add(self, cname: str) -> None: if cname in self._containers: - _LOGGER.error("[%s] %s: Container already monitored", self._instance, cname) - return + # The container was recreated (new id) while we still hold the + # old monitor. Replace the stale monitor instead of bailing out + # with 'Container already monitored'. + _LOGGER.warning( + "[%s] %s: Container already monitored, replacing stale monitor", + self._instance, + cname, + ) + await self._container_remove(cname) _LOGGER.debug("[%s] %s: Starting Container Monitor", self._instance, cname) # Create our Docker Container API self._containers[cname] = DockerContainerAPI( - self._config, self._api, cname, atInit=False + self._config, self._api, cname, atInit=False, parent=self ) # We should wait until container is attached @@ -889,9 +913,12 @@ def __init__( api: aiodocker.Docker, cname: str, atInit=True, + parent: "DockerAPI | None" = None, ): self._config = config self._api = api + self._parent: "DockerAPI | None" = parent + self._gone = 0 self._instance: str = config[CONF_NAME] self._memChange: int = config[CONF_MEMORYCHANGE] self._name = cname @@ -966,6 +993,55 @@ async def _initGetContainer(self) -> bool: return True + ############################################################# + async def _handle_container_gone(self, error: str) -> None: + """Handle a 404 from the Docker API for the monitored container. + + The monitor keeps a handle for the container id it was created with. + When the container is recreated (Watchtower, compose up -d, ... + ) that id no longer exists while the new container has a new id but + the same name. Try to re-attach to a container with the same name; if + there is none, ask the DockerAPI instance to drop this monitor so a + later create event can add it back. + """ + try: + self._container = await self._api.containers.get(self._name) + except aiodocker.exceptions.DockerError: + self._gone += 1 + if self._gone >= 2 and self._parent is not None: + _LOGGER.info( + "[%s] %s: Container not available anymore (3a) (%s). " + "Container does not exist anymore, dropping monitor", + self._instance, + self._name, + error, + ) + await self._parent.on_container_gone(self._name) + else: + _LOGGER.error( + "[%s] %s: Container not available anymore (3a) (%s). Retry in %d seconds", + self._instance, + self._name, + error, + self._retry_interval, + ) + except Exception as err: + self._gone = 0 + _LOGGER.error( + "[%s] %s: Container not available anymore (3b) (%s). Retry in %d seconds", + self._instance, + self._name, + str(err), + self._retry_interval, + ) + else: + self._gone = 0 + _LOGGER.info( + "[%s] %s: Re-attached to recreated container", + self._instance, + self._name, + ) + ############################################################# def request_refresh(self) -> None: """Wake this container's loop to fetch immediately.""" @@ -1017,13 +1093,16 @@ async def _run(self) -> None: ) break except aiodocker.exceptions.DockerError as err: - _LOGGER.error( - "[%s] %s: Container not available anymore (3a) (%s). Retry in %d seconds", - self._instance, - self._name, - str(err), - self._retry_interval, - ) + if err.status == 404: + await self._handle_container_gone(str(err)) + else: + _LOGGER.error( + "[%s] %s: Container not available anymore (3a) (%s). Retry in %d seconds", + self._instance, + self._name, + str(err), + self._retry_interval, + ) except asyncio.TimeoutError as err: _LOGGER.error( "[%s] %s: Container not available anymore (3d) TimeoutError. Retry in %d seconds",