Merge pull request #395 from ChrispyBacon-dev/unstable

Unstable merge v3.1.4

.github/workflows/docker-agent.yml

@@ -0,0 +1,137 @@
+name: Docker Image - DockFlare Agent
+
+on:
+  push:
+    branches:
+      - "stable"
+      - "unstable"
+      - "dev"
+      - "dockflare-mail"
+    tags:
+      - "v*.*.*"
+      - "v*.*.*-*"
+    paths:
+      - "dockflare-agent/**"
+      - ".github/workflows/docker-agent.yml"
+  pull_request:
+    branches:
+      - "stable"
+      - "unstable"
+      - "dev"
+      - "dockflare-mail"
+    paths:
+      - "dockflare-agent/**"
+      - ".github/workflows/docker-agent.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: docker-agent-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build_self_hosted:
+    runs-on: self-hosted
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Log in to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v6
+        with:
+          images: alplat/dockflare-agent
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{version}},suffix=-{{premajor}}-{{prerelease}}
+            type=ref,event=branch,pattern=stable,value=latest
+            type=ref,event=branch,pattern=stable,value=stable
+            type=ref,event=branch,pattern=unstable,value=unstable
+            type=ref,event=branch,pattern=dev,value=dev
+            type=sha,format=short
+            type=ref,event=tag
+            type=ref,event=branch
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v7
+        with:
+          context: ./dockflare-agent
+          file: ./dockflare-agent/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,scope=dockflare-agent
+          cache-to: type=gha,mode=max,scope=dockflare-agent
+
+  build_github_hosted_fallback:
+    needs: build_self_hosted
+    if: failure() || cancelled()
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Log in to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v6
+        with:
+          images: alplat/dockflare-agent
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{version}},suffix=-{{premajor}}-{{prerelease}}
+            type=ref,event=branch,pattern=stable,value=latest
+            type=ref,event=branch,pattern=stable,value=stable
+            type=ref,event=branch,pattern=unstable,value=unstable
+            type=ref,event=branch,pattern=dev,value=dev
+            type=sha,format=short
+            type=ref,event=tag
+            type=ref,event=branch
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v7
+        with:
+          context: ./dockflare-agent
+          file: ./dockflare-agent/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,scope=dockflare-agent
+          cache-to: type=gha,mode=max,scope=dockflare-agent

.github/workflows/docker-image.yml

@@ -22,9 +22,6 @@ on:
     paths:
       - "dockflare/**"
 
-env:
-  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
-
 permissions:
   contents: read
 
@@ -32,30 +29,30 @@ jobs:
   build_self_hosted:
     runs-on: self-hosted
 
-    timeout-minutes: 3
+    timeout-minutes: 30
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Log in to Docker Hub
         if: github.event_name == 'push'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           images: alplat/dockflare
           tags: |
@@ -71,41 +68,44 @@ jobs:
 
       - name: Build and Push Docker Image
         id: build_and_push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v7
         with:
           context: ./dockflare
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name == 'push' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,scope=dockflare
+          cache-to: type=gha,mode=max,scope=dockflare
 
   build_github_hosted_fallback:
     needs: build_self_hosted
     if: failure() || cancelled()
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Log in to Docker Hub
         if: github.event_name == 'push'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           images: alplat/dockflare
           tags: |
@@ -121,10 +121,12 @@ jobs:
 
       - name: Build and Push Docker Image
         id: build_and_push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v7
         with:
           context: ./dockflare
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name == 'push' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,scope=dockflare
+          cache-to: type=gha,mode=max,scope=dockflare

.github/workflows/docker-mail-manager.yml

@@ -22,9 +22,6 @@ on:
     paths:
       - "mail-manager/**"
 
-env:
-  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
-
 permissions:
   contents: read
 
@@ -32,30 +29,30 @@ jobs:
   build_self_hosted:
     runs-on: self-hosted
 
-    timeout-minutes: 3
+    timeout-minutes: 30
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Log in to Docker Hub
         if: github.event_name == 'push'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           images: alplat/dockflare-mail-manager
           tags: |
@@ -71,41 +68,44 @@ jobs:
 
       - name: Build and Push Docker Image
         id: build_and_push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v7
         with:
           context: ./mail-manager
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name == 'push' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,scope=dockflare-mail-manager
+          cache-to: type=gha,mode=max,scope=dockflare-mail-manager
 
   build_github_hosted_fallback:
     needs: build_self_hosted
     if: failure() || cancelled()
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Log in to Docker Hub
         if: github.event_name == 'push'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           images: alplat/dockflare-mail-manager
           tags: |
@@ -121,10 +121,12 @@ jobs:
 
       - name: Build and Push Docker Image
         id: build_and_push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v7
         with:
           context: ./mail-manager
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name == 'push' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,scope=dockflare-mail-manager
+          cache-to: type=gha,mode=max,scope=dockflare-mail-manager

.github/workflows/docker-webmail.yml

@@ -22,9 +22,6 @@ on:
     paths:
       - "webmail/**"
 
-env:
-  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
-
 permissions:
   contents: read
 
@@ -32,30 +29,30 @@ jobs:
   build_self_hosted:
     runs-on: self-hosted
 
-    timeout-minutes: 3
+    timeout-minutes: 30
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Log in to Docker Hub
         if: github.event_name == 'push'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           images: alplat/dockflare-webmail
           tags: |
@@ -71,41 +68,44 @@ jobs:
 
       - name: Build and Push Docker Image
         id: build_and_push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v7
         with:
           context: ./webmail
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name == 'push' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,scope=dockflare-webmail
+          cache-to: type=gha,mode=max,scope=dockflare-webmail
 
   build_github_hosted_fallback:
     needs: build_self_hosted
     if: failure() || cancelled()
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           fetch-depth: 0
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v4
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Log in to Docker Hub
         if: github.event_name == 'push'
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v6
         with:
           images: alplat/dockflare-webmail
           tags: |
@@ -121,10 +121,12 @@ jobs:
 
       - name: Build and Push Docker Image
         id: build_and_push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v7
         with:
           context: ./webmail
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name == 'push' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,scope=dockflare-webmail
+          cache-to: type=gha,mode=max,scope=dockflare-webmail

.gitignore

@@ -84,6 +84,9 @@ $RECYCLE.BIN/
 .env
 .claude
 .docs
+/dockflare/tests/
+dockflare-agent/overview.json
+dockflare-agent/data/
 data/state.json
 *.tmp
 *.bak
@@ -92,4 +95,4 @@ data/state.json
 logs/
 DockFlare-Agent-prd/
 unstable-compose.yml
-unstable-local.yml
+unstable-local.yml

CHANGELOG.md

@@ -2,6 +2,22 @@
 
 All notable changes to this project will be documented in this file.
 
+## [Unreleased]
+
+### Added
+- **Apprise notifications:** Added opt-in, encrypted notification destinations for rule lifecycle, Cloudflare failures, Docker listener failures, Agent enrollment, decommission lifecycle, Agent health, tunnel health, and explicit Access Policy management. Event controls are grouped by area, with routine administrative events opt-in and failure/terminal observability enabled by default. Delivery uses a bounded background queue with cooldowns, startup and intentional-action suppression, redacted settings/status output, and authenticated test delivery. Addresses [#302](https://github.com/ChrispyBacon-dev/DockFlare/issues/302).
+- **Safe Agent decommissioning:** Replaced immediate Agent deletion with a durable prepare, ownership-safe Cloudflare cleanup, and final Agent shutdown workflow. Decommissioned Agents persist a local tombstone, stop their managed tunnel without Docker delete permission, revoke their API key only after final acknowledgement, and cannot silently recreate the tunnel after restart. The Agents UI now shows retryable progress and provides separately acknowledged host cleanup commands that remove the Agent Docker deployment while preserving the shared external `cloudflare-net` network.
+
+### Fixed
+- **Frontend dependency refresh:** Updated compatible PostCSS, Axios, DOMPurify, Vue, and transitive dependencies.
+- **UI-overridden container lifecycle:** Recreated Docker and Agent-managed containers now reactivate their existing rules, refresh only their runtime container association, and retain UI-controlled routing, origin, tunnel, and Access settings. Fixes [#388](https://github.com/ChrispyBacon-dev/DockFlare/issues/388).
+  - Added stable source-rule identity so UI-renamed container-backed rules remain associated with their originating Docker or Agent label binding without creating duplicates.
+  - Kept pending rules in tunnel ingress for the full deletion grace period, with generation-aware cleanup, per-tunnel serialization, DNS ownership checks, and durable synchronization retries protecting routes during failures and event races.
+  - Made full-rule and Access-policy ownership independently reversible for both Docker and Agent sources. Agent reverts no longer depend on a local Docker socket and apply stored inventory only when it matches the rule's current container and source binding; otherwise DockFlare safely waits for the next authenticated Agent observation.
+  - Added content fingerprinting to the main browser script so updated rule-editor logic is loaded after an upgrade instead of a stale cached asset. The editor continues to submit the immutable source rule key rather than inferring ownership from editable hostname or path fields.
+  - Strengthened Agent reporting with master-issued sessions, ordered event and report streams, explicit inventory completeness, bounded payloads, filtered DockFlare label namespaces, normalized Docker lifecycle events, and safe reconnect behavior.
+- **Overview API secret exposure and read side effects:** Removed reusable tunnel, Agent session, Agent API-key, and account material from overview/list responses. Agent-key actions now use stable non-authorizing references, while explicit key-generation and deployment flows retain their intended one-time credential behavior. `GET /api/v2/overview` no longer backfills or persists rule state.
+
 ## [v3.1.4] - 2026-08-17
 
 ### Fixed

README.MD

@@ -313,6 +313,8 @@ Run a central master with agents on remote Docker hosts. Agents stream host-loca
 
 Multi-agent setup guide: [https://dockflare.app/docs/multi-server-agent](https://dockflare.app/docs/multi-server-agent)
 
+Agent source and standalone deployment files: [`dockflare-agent/`](dockflare-agent/)
+
 ## Access Control Model
 
 DockFlare uses Access Groups as the primary abstraction for reusable access intent.

dockflare-agent/.dockerignore

@@ -0,0 +1,11 @@
+.DS_Store
+.env
+.git
+.github
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.venv/
+data/
+overview.json
+tests/

dockflare-agent/Dockerfile

@@ -0,0 +1,27 @@
+FROM python:3.11.6-slim
+
+ARG DOCKFLARE_UID=65532
+ARG DOCKFLARE_GID=65532
+
+WORKDIR /app
+
+ENV HEALTH_CHECK_PORT=8080
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY ./app .
+
+RUN addgroup --system --gid ${DOCKFLARE_GID} dockflare \
+    && adduser --system --uid ${DOCKFLARE_UID} --ingroup dockflare dockflare \
+    && mkdir -p /app/data \
+    && chown -R dockflare:dockflare /app
+
+USER dockflare:dockflare
+
+EXPOSE 8080
+
+HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
+    CMD python -c "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ.get('HEALTH_CHECK_PORT', '8080') + '/health', timeout=3)"
+
+CMD ["python", "main.py"]

dockflare-agent/README.md

@@ -0,0 +1,259 @@
+<p align="center">
+  <a href="https://dockflare.app" title="DockFlare Agent">
+    <img src="https://raw.githubusercontent.com/ChrispyBacon-dev/DockFlare/main/images/bannertr.png" width="480" alt="DockFlare Banner" />
+  </a>
+</p>
+
+> **Note:** This directory contains the **DockFlare Agent**, which is designed to work as a worker node in a multi-server setup. It is not a standalone project and requires the main [DockFlare application](https://github.com/ChrispyBacon-dev/DockFlare) to function.
+
+<h1 align="center">DockFlare Agent</h1>
+
+<p align="center">
+  <em>Lightweight workers that report Docker changes, run cloudflared tunnels, and obey the DockFlare Master.</em>
+</p>
+
+<p align="center">
+  <a href="https://hub.docker.com/r/alplat/dockflare-agent"><img src="https://img.shields.io/docker/pulls/alplat/dockflare-agent?style=for-the-badge" alt="Docker Pulls"></a>
+  <a href="https://github.com/ChrispyBacon-dev/DockFlare/tree/stable/dockflare-agent"><img src="https://img.shields.io/badge/Status-Beta-blue?style=for-the-badge" alt="Status"></a>
+  <a href="https://www.python.org/"><img src="https://img.shields.io/badge/Made%20with-Python-1f425f.svg?style=for-the-badge" alt="Python"></a>
+  <a href="../LICENSE.MD"><img src="https://img.shields.io/badge/License-GPL--3.0-blue.svg?style=for-the-badge" alt="License"></a>
+  <a href="#"><img src="https://img.shields.io/badge/Swiss_Made-FFFFFF?style=for-the-badge&labelColor=FF0000&logo=data:image/svg%2bxml;base64,PHN2ZyB2ZXJzaW9uPSIxIiB3aWR0aD0iNTEyIiBoZWlnaHQ9IjUxMiIgdmlld0JveD0iMCAwIDMyIDMyIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxyZWN0IHdpZHRoPSIzMiIgaGVpZHRoPSIzMiIgZmlsbD0idHJhbnNwYXJlbnQiLz4KICA8cGF0aCBkPSJtMTMgNmg2djdoN3Y2aC03djdoLTZ2LTdoLTd2LTZoN3oiIGZpbGw9IiNmZmYiLz4KPC9zdmc+" alt="Swiss Made"></a>
+</p>
+
+<p align="center">
+  <a href="https://dockflare.app">🌐 Website</a> ·
+  <a href="https://dockflare.app/docs/agent">📚 Agent Docs</a> ·
+  <a href="https://github.com/ChrispyBacon-dev/DockFlare/tree/stable/dockflare-agent">💻 Source</a> ·
+  <a href="https://github.com/ChrispyBacon-dev/DockFlare/issues">🐛 Report a Bug</a> ·
+  <a href="https://github.com/sponsors/ChrispyBacon-dev">❤️ Sponsor</a>
+</p>
+
+---
+
+## Overview
+
+DockFlare 3.0 introduces a distributed control plane: a central **DockFlare Master** owns desired ingress and policy state, while lightweight **DockFlare Agents** observe workloads and execute master-issued runtime commands. The agent is a headless Python service that watches Docker events, reports observations to the master, and supervises a dedicated `cloudflared` container.
+
+Deploy agents on any Docker-capable host to extend DockFlare beyond a single server. Each agent reports host-local observations and health while the master remains authoritative for ingress, tunnel assignment, lifecycle, DNS, and Access decisions. Existing `cloudflared` processes can continue serving the last applied configuration during a temporary master outage.
+
+### Highlights
+
+- **Distributed ingress** – manage tunnels on remote hosts without exposing raw credentials.
+- **Real-time visibility** – agents stream lifecycle events, periodic status reports, and tunnel metrics back to the master.
+- **Least privilege** – per-agent API keys can be rotated or revoked without affecting the rest of the fleet.
+- **Resilient execution** – cached tunnel state lets agents ride out transient master outages.
+
+<details>
+  <summary><strong>🚀 Quick Start: Deploying the Agent</strong></summary>
+
+### Docker Compose (Recommended)
+
+To deploy the agent, create the following two files in the same directory.
+
+**1. `docker-compose.yml`**
+
+```yaml
+version: '3.8'
+
+services:
+  docker-socket-proxy:
+    image: tecnativa/docker-socket-proxy:v0.4.1
+    container_name: docker-socket-proxy
+    restart: unless-stopped
+    environment:
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - CONTAINERS=1
+      - EVENTS=1
+      - NETWORKS=1
+      - IMAGES=1
+      - POST=1
+      - PING=1
+      - EXEC=1
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    networks:
+      - dockflare-internal
+
+  dockflare-agent:
+    image: alplat/dockflare-agent:latest
+    container_name: dockflare-agent
+    restart: unless-stopped
+    env_file:
+      - .env
+    environment:
+      - DOCKER_HOST=${DOCKER_HOST:-tcp://docker-socket-proxy:2375}
+      - TZ=${TZ:-UTC}
+      - LOG_LEVEL=${LOG_LEVEL:-info}
+    volumes:
+      - agent_data:/app/data
+    depends_on:
+      - docker-socket-proxy
+    networks:
+      - cloudflare-net
+      - dockflare-internal
+
+volumes:
+  agent_data:
+
+networks:
+  cloudflare-net:
+    name: cloudflare-net
+    external: true
+  dockflare-internal:
+    name: dockflare-internal
+```
+
+**2. `.env` file**
+
+Next, create a `.env` file in the same directory. This file provides the configuration values for the agent service defined above. Refer to the **Configuration** section below for more details on each variable.
+
+```
+DOCKFLARE_MASTER_URL=https://dockflare.example.com
+DOCKFLARE_API_KEY=agent_api_key_goes_here
+DOCKER_HOST=tcp://docker-socket-proxy:2375
+AGENT_DISPLAY_NAME=Production Server
+# control the docker image used for the managed cloudflared tunnel (accepts repo:tag or repo@sha256:<digest>)
+CLOUDFLARED_IMAGE=cloudflare/cloudflared:2025.9.0
+LOG_LEVEL=info
+TZ=Europe/Zurich
+```
+
+Once both files are in place, run `docker-compose up -d` to start the agent.
+
+- The proxy limits the Docker API surface the agent can reach; only the variables set to `1` are exposed.
+- Granting `IMAGES=1` allows the agent to pull the managed `cloudflared` image.
+- Provide a persistent volume for `/app/data` to ensure agent identity survives restarts.
+- Ensure the external network (`cloudflare-net` by default) exists before starting.
+
+</details>
+
+---
+
+## Architecture Snapshot
+
+| Component | Responsibility |
+|-----------|----------------|
+| **DockFlare Master** | Stores desired state, reconciles DNS/Access policies, issues commands via HTTPS. |
+| **Redis** | Provides the backplane for heartbeats, command queues, and shared caches. |
+| **DockFlare Agent** | Runs on the managed host, watches Docker events, manages `cloudflared`, and reports status. |
+| **cloudflared** | The Cloudflare tunnel process launched and supervised by the agent. |
+
+### Repository Layout
+
+```
+dockflare-agent/
+├── app/
+│   ├── __init__.py
+│   ├── cloudflare_api.py
+│   └── main.py
+├── .dockerignore
+├── Dockerfile
+├── docker-compose.yml
+├── env-example
+├── requirements.txt
+└── README.md
+```
+
+---
+
+## Runtime Flow
+
+1. **Bootstrapping** – environment variables are loaded, logging is configured, and cached agent identity/tunnel data are restored from `/app/data`.
+2. **Registration** – the agent authenticates with the master using `DOCKFLARE_API_KEY`, receives (or refreshes) its Agent ID, and persists it locally.
+3. **Thread fan-out** – shared Docker client powers four background workers:
+   - `manage_tunnels` polls for commands (`start_tunnel`, `stop_tunnel`, `update_tunnel_config`).
+   - `periodic_status_reporter` emits heartbeats and summaries of labelled containers every `REPORT_INTERVAL_SECONDS`.
+   - `listen_for_docker_events` streams container lifecycle events for `dockflare.enable=true` workloads.
+   - `tunnel_health_monitor` verifies the managed `cloudflared` container remains healthy.
+4. **Shutdown** – `cleanup()` stops and removes the managed tunnel container before the agent exits.
+
+### Master authority and protocol compatibility
+
+The DockFlare Master is authoritative for rule ownership, lifecycle decisions, tunnel assignment, ingress, DNS, and Access configuration. Agents report filtered Docker observations and execute runtime commands; they do not calculate desired Cloudflare configuration.
+
+Current Agents advertise protocol versions 2 and 1 during registration. A compatible master selects the protocol and issues a new process-session identifier. Protocol v2 sends independently ordered lifecycle and inventory streams, and a complete inventory is accepted only after the Agent finishes a full Docker scan. If scanning fails, the Agent sends an explicitly incomplete report so the master cannot infer that containers disappeared. Deploy the compatible master before upgrading Agents; legacy Agents remain positive-observation-only during a mixed-version rollout.
+
+### Cloudflare Helper Module
+
+`app/cloudflare_api.py` provides the thin wrapper that the agent uses to proxy Cloudflare API calls through the master:
+
+- `get_account_id(master_url, api_key)` – resolves the Cloudflare account the master exposes to agents.
+- `generate_ingress_rules(rules)` – converts desired ingress records into a tunnel configuration payload.
+- `update_tunnel_config(master_url, api_key, tunnel_id, ingress_rules)` – pushes ingress updates via the master’s API.
+
+---
+
+## Requirements
+
+- DockFlare Master **v3.0 or later** running with Redis and HTTPS enabled.
+- Docker Engine on every host that will run the agent.
+- Network reachability from the agent to the master (public HTTPS or a private network/VPN).
+- Cloudflare account + API token (managed by the master; agents never handle raw Cloudflare credentials).
+
+---
+
+## Configuration
+
+The agent is configured using environment variables, typically through the `.env` file shown in the deployment guide.
+
+| Variable | Required | Description |
+|----------|----------|-------------|
+| `DOCKFLARE_MASTER_URL` | ✅ | Base URL of the DockFlare Master (`https://dockflare.example.com`). |
+| `DOCKFLARE_API_KEY` | ✅ | Agent API key generated in the master UI (`Agents → Generate Key`). |
+| `CLOUDFLARED_IMAGE` | ✅ | Preferred Cloudflared release (`cloudflare/cloudflared:2025.9.0`) or digest (`cloudflare/cloudflared@sha256:...`). |
+| `DOCKER_HOST` | ✅ | Address of the Docker socket proxy (`tcp://docker-socket-proxy:2375`). |
+| `AGENT_DISPLAY_NAME` | ❌ | Human-readable name for the agent (`Production Server`, `NAS Server`). Falls back to `agent-{8chars}` if not set. |
+| `CLOUDFLARED_NETWORK_NAME` | ❌ | Docker network used for the managed tunnel (`cloudflare-net` by default). |
+| `LOG_LEVEL` | ❌ | Python logging level (`INFO` by default). |
+| `HEALTH_CHECK_PORT` | ❌ | Internal health endpoint port (`8080` by default). |
+| `REPORT_INTERVAL_SECONDS` | ❌ | Cadence for status reports (defaults to `30`). |
+| `TZ` | ❌ | Host timezone exposed to the container (`UTC` by default). |
+
+The agent persists lightweight state inside `/app/data`:
+
+- `agent_id.txt` – the master-issued identifier for the node.
+- `tunnel_state.json` – cached tunnel token, ID, name, and desired state.
+
+Bind-mount a volume to `/app/data` in production so identity survives container restarts.
+
+---
+
+## Security Model & Hardening
+
+- **Master API key** protects administrative APIs; only expose it when enrolling trusted agents.
+- **Per-agent API keys** are revocable—delete the key in the master UI to immediately cut off a compromised host.
+- **Transport security** – front the master with HTTPS (or Cloudflare Access) so agent traffic is encrypted end-to-end.
+- **Redis** should reside on a trusted network segment and require authentication when deployed outside a lab environment.
+- **Docker access** is mediated through the bundled socket proxy so the agent can only list containers, stream events, manage networks, and operate its tunnel container.
+- **Least privilege container** – the agent image runs as the `dockflare` user (UID/GID 65532); no root processes remain once start-up is complete.
+
+Recommended practices:
+
+1. Store agent keys in a password manager and rotate them regularly.
+2. Use dedicated Cloudflare tunnels per agent for blast-radius isolation.
+3. Monitor heartbeat gaps on the master’s Agents page; prune offline nodes promptly.
+
+---
+
+## Troubleshooting
+
+| Symptom | Resolution |
+|---------|------------|
+| Agent stuck in `pending` | Verify the API key, ensure the agent can reach the master, and enrol it from the UI. |
+| Commands never clear | Confirm Redis connectivity and that host clocks are in sync. |
+| DNS or Access policies not updating | Check agent logs (`docker logs dockflare-agent`) and confirm cloudflared is running. |
+| Heartbeat offline | Inspect network path and TLS configuration between agent and master. |
+
+---
+
+## Next Steps
+
+- Follow the DockFlare Master [Quick Start](https://dockflare.app/docs) to prepare the control plane.
+- Generate an agent key in the master UI and deploy this container on remote hosts.
+- Track upcoming releases from the [DockFlare Agent Docker Hub repository](https://hub.docker.com/r/alplat/dockflare-agent).
+
+---
+
+## License
+
+DockFlare Agent is open-source software licensed under the [GPL-3.0 license](../LICENSE.MD).

dockflare-agent/app/__init__.py

@@ -0,0 +1 @@
+"""DockFlare Agent package."""

dockflare-agent/app/cloudflare_api.py

@@ -0,0 +1,59 @@
+import requests
+import logging
+
+def update_tunnel_config(master_url, api_key, tunnel_id, ingress_rules):
+    """
+    Updates the Cloudflare tunnel configuration for the agent.
+    """
+    if not all([master_url, api_key, tunnel_id]):
+        logging.error("Missing master_url, api_key, or tunnel_id for tunnel config update.")
+        return False
+
+    endpoint = f"/accounts/{get_account_id(master_url, api_key)}/cfd_tunnel/{tunnel_id}/configurations"
+    url = f"{master_url.rstrip('/')}{endpoint}"
+    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
+    payload = {"config": {"ingress": ingress_rules}}
+
+    try:
+        response = requests.put(url, json=payload, headers=headers, timeout=30)
+        response.raise_for_status()
+        logging.info(f"Successfully updated tunnel config for {tunnel_id}")
+        return True
+    except requests.exceptions.RequestException as e:
+        logging.error(f"Error updating tunnel config for {tunnel_id}: {e}")
+        return False
+
+def generate_ingress_rules(rules):
+    """
+    Generates ingress rules for Cloudflare tunnel from agent rules.
+    """
+    ingress = []
+    for rule_key, rule in rules.items():
+        if rule.get("status") == "active":
+            entry = {"hostname": rule["hostname"], "service": rule["service"]}
+            if rule.get("path"):
+                entry["path"] = rule["path"]
+            ingress.append(entry)
+    ingress.append({"service": "http_status:404"})
+    return ingress
+
+def get_account_id(master_url, api_key):
+    """
+    Retrieves the Cloudflare account ID from the master.
+    """
+    endpoint = "/accounts"
+    url = f"{master_url.rstrip('/')}{endpoint}"
+    headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
+
+    try:
+        response = requests.get(url, headers=headers, timeout=15)
+        response.raise_for_status()
+        data = response.json()
+        if data.get("success") and data.get("result"):
+            return data["result"][0]["id"]
+        else:
+            logging.error(f"Failed to get account ID from master: {data}")
+            return None
+    except requests.exceptions.RequestException as e:
+        logging.error(f"Error getting account ID from master: {e}")
+        return None

dockflare-agent/app/decommission.py

@@ -0,0 +1,147 @@
+"""Local, restart-safe Agent decommission primitives."""
+
+import json
+import os
+import re
+import tempfile
+import time
+from datetime import datetime, timezone
+from threading import Thread
+
+import docker
+
+
+TOMBSTONE_PATH = os.getenv("DECOMMISSION_TOMBSTONE_PATH", "/app/data/decommission.json")
+TUNNEL_CONTAINER_NAME = "dockflare-agent-tunnel"
+AGENT_CONTAINER_NAME = "dockflare-agent"
+IMAGE_REFERENCE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@-]{0,254}$")
+
+
+def _iso_now():
+    return datetime.now(timezone.utc).isoformat()
+
+
+def load_tombstone(path=TOMBSTONE_PATH):
+    if not os.path.exists(path):
+        return None
+    try:
+        with open(path, "r", encoding="utf-8") as handle:
+            value = json.load(handle)
+        if not isinstance(value, dict) or value.get("phase") not in {"prepared", "finalized"}:
+            return None
+        return value
+    except (OSError, ValueError):
+        return None
+
+
+def persist_tombstone(operation_id, agent_id, phase, path=TOMBSTONE_PATH):
+    existing = load_tombstone(path) or {}
+    if existing and (
+        existing.get("operation_id") != operation_id
+        or existing.get("agent_id") != agent_id
+        or (existing.get("phase") == "finalized" and phase != "finalized")
+    ):
+        raise ValueError("decommission_tombstone_conflict")
+    now = _iso_now()
+    value = {
+        "schema_version": 1,
+        "operation_id": operation_id,
+        "agent_id": agent_id,
+        "phase": phase,
+        "requested_self_action": "stop",
+        "prepared_at": existing.get("prepared_at") or now,
+        "finalized_at": now if phase == "finalized" else None,
+    }
+    directory = os.path.dirname(path)
+    os.makedirs(directory, exist_ok=True)
+    temp_path = None
+    try:
+        with tempfile.NamedTemporaryFile("w", dir=directory, delete=False, encoding="utf-8") as handle:
+            json.dump(value, handle)
+            handle.flush()
+            os.fsync(handle.fileno())
+            temp_path = handle.name
+        os.chmod(temp_path, 0o600)
+        os.replace(temp_path, path)
+        return value
+    except Exception:
+        if temp_path and os.path.exists(temp_path):
+            try:
+                os.remove(temp_path)
+            except OSError:
+                pass
+        raise
+
+
+def normalized_container_image(container):
+    try:
+        value = (container.attrs.get("Config") or {}).get("Image")
+    except (AttributeError, TypeError):
+        value = None
+    if isinstance(value, str):
+        candidate = value.strip()
+        if IMAGE_REFERENCE_RE.fullmatch(candidate):
+            return candidate
+    return None
+
+
+def stop_tunnel_container(client):
+    """Stop, but never delete, the one fixed Agent-managed tunnel container."""
+    try:
+        container = client.containers.get(TUNNEL_CONTAINER_NAME)
+    except docker.errors.NotFound:
+        return "absent", None
+    if getattr(container, "name", None) != TUNNEL_CONTAINER_NAME:
+        return "failed", None
+    image = normalized_container_image(container)
+    try:
+        container.reload()
+        if getattr(container, "status", None) not in {"exited", "dead", "created"}:
+            container.stop(timeout=15)
+        container.reload()
+        return ("stopped", image) if getattr(container, "status", None) != "running" else ("failed", image)
+    except docker.errors.NotFound:
+        return "absent", image
+    except Exception:
+        return "failed", image
+
+
+def resolve_self_container(client, runtime_id=None):
+    candidate_id = (runtime_id or os.getenv("HOSTNAME") or "").strip()
+    if not candidate_id or not re.fullmatch(r"[a-fA-F0-9]{12,64}", candidate_id):
+        return None
+    try:
+        container = client.containers.get(candidate_id)
+    except Exception:
+        return None
+    full_id = str(getattr(container, "id", ""))
+    if not full_id.startswith(candidate_id) and not candidate_id.startswith(full_id):
+        return None
+    labels = getattr(container, "labels", None) or {}
+    service = labels.get("com.docker.compose.service")
+    name = getattr(container, "name", None)
+    if service != "dockflare-agent" and name != AGENT_CONTAINER_NAME:
+        return None
+    return container
+
+
+def self_image_reference(client, runtime_id=None):
+    container = resolve_self_container(client, runtime_id)
+    return normalized_container_image(container) if container else None
+
+
+def schedule_self_stop(client, runtime_id=None, delay_seconds=2):
+    """Schedule an explicit Docker stop after the final ack response is received."""
+    container = resolve_self_container(client, runtime_id)
+    if not container:
+        return False
+
+    def stop_later():
+        time.sleep(delay_seconds)
+        try:
+            container.stop(timeout=15)
+        except Exception:
+            return
+
+    Thread(target=stop_later, name="AgentSelfStop", daemon=True).start()
+    return True

dockflare-agent/app/main.py

@@ -0,0 +1,929 @@
+import os
+import re
+import time
+import json
+import docker
+import requests
+import logging
+import tempfile
+import http.server
+import socketserver
+import random
+from threading import Thread, Lock
+from datetime import datetime, timezone
+from dotenv import load_dotenv
+import cloudflare_api
+import decommission as decommission_runtime
+
+DEFAULT_CLOUDFLARED_IMAGE = "cloudflare/cloudflared:2025.9.0"
+_SHA256_HEX_RE = re.compile(r"^[A-Fa-f0-9]{64}$")
+
+
+def is_dockflare_enabled(labels):
+    """Check if container has DockFlare enabled using new or legacy labels."""
+    if not labels:
+        return False
+    values = (labels.get("dockflare.enable"), labels.get("cloudflare.tunnel.enable"))
+    return any(isinstance(value, str) and value.lower() in {"true", "1", "t", "yes"} for value in values)
+
+
+def _normalize_cloudflared_image(raw_value, default_image):
+    """Sanitize the configured cloudflared image reference."""
+    if not raw_value:
+        return default_image
+
+    candidate = raw_value.strip()
+    if not candidate:
+        logging.warning("CLOUDFLARED_IMAGE is blank after trimming; falling back to default %s", default_image)
+        return default_image
+
+    # Split on whitespace to drop inline comments or accidental extra tokens.
+    candidate = candidate.split()[0]
+
+    if "#" in candidate:
+        candidate = candidate.split("#", 1)[0].strip()
+        if not candidate:
+            logging.warning("CLOUDFLARED_IMAGE only contained a comment; falling back to default %s", default_image)
+            return default_image
+
+    if "@sha256:" in candidate:
+        repository, digest = candidate.split("@sha256:", 1)
+        if not repository:
+            logging.error("CLOUDFLARED_IMAGE is missing the repository before @sha256; falling back to default %s", default_image)
+            return default_image
+        if not _SHA256_HEX_RE.fullmatch(digest):
+            logging.error("CLOUDFLARED_IMAGE digest is not a valid 64 char hex string; falling back to default %s", default_image)
+            return default_image
+        return f"{repository}@sha256:{digest.lower()}"
+
+    return candidate
+
+# Basic Logging Configuration (configurable via LOG_LEVEL env var; default INFO)
+_LOG_LEVEL_STR = os.getenv('LOG_LEVEL', 'INFO').upper()
+_LOG_LEVEL = getattr(logging, _LOG_LEVEL_STR, logging.INFO)
+logging.basicConfig(level=_LOG_LEVEL, format='%(asctime)s - %(levelname)s - %(message)s')
+logging.getLogger().info(f"Logging initialized at level: {_LOG_LEVEL_STR}")
+
+# Load environment variables from .env file
+load_dotenv()
+
+MASTER_URL = os.getenv("DOCKFLARE_MASTER_URL")
+API_KEY = os.getenv("DOCKFLARE_API_KEY")
+AGENT_VERSION = os.getenv("DOCKFLARE_AGENT_VERSION", "1.1.0")
+CF_ACCESS_CLIENT_ID = os.getenv("CF_ACCESS_CLIENT_ID", "")
+CF_ACCESS_CLIENT_SECRET = os.getenv("CF_ACCESS_CLIENT_SECRET", "")
+
+
+def get_headers(content_type=True):
+    headers = {"Authorization": f"Bearer {API_KEY}"}
+    if content_type:
+        headers["Content-Type"] = "application/json"
+    if CF_ACCESS_CLIENT_ID:
+        headers["CF-Access-Client-Id"] = CF_ACCESS_CLIENT_ID
+        headers["CF-Access-Client-Secret"] = CF_ACCESS_CLIENT_SECRET
+    return headers
+CLOUDFLARED_NETWORK_NAME = os.getenv("CLOUDFLARED_NETWORK_NAME", "cloudflare-net")
+CLOUDFLARED_IMAGE = _normalize_cloudflared_image(os.getenv("CLOUDFLARED_IMAGE"), DEFAULT_CLOUDFLARED_IMAGE)
+if CLOUDFLARED_IMAGE != DEFAULT_CLOUDFLARED_IMAGE:
+    logging.info(f"Using configured cloudflared image: {CLOUDFLARED_IMAGE}")
+else:
+    logging.info(f"Using default cloudflared image: {DEFAULT_CLOUDFLARED_IMAGE}")
+AGENT_ID_FILE = "/app/data/agent_id.txt"
+TUNNEL_STATE_FILE = "/app/data/tunnel_state.json"
+AGENT_ID = None  # Will be assigned by the master
+PROTOCOL_VERSION = None
+AGENT_SESSION_ID = None
+_event_sequence = 0
+_report_sequence = 0
+_protocol_lock = Lock()
+_event_send_lock = Lock()
+_report_send_lock = Lock()
+_registration_lock = Lock()
+
+# --- Tunnel Management Globals ---
+tunnel_container = None
+current_tunnel_token = None
+current_tunnel_id = None
+current_tunnel_version = None
+current_tunnel_name = None
+desired_tunnel_state = "unknown"
+decommission_tombstone = None
+
+# --- Health / Monitoring Globals ---
+thread_health_status = {}
+last_successful_master_contact = None
+
+
+def _configured_health_port():
+    try:
+        value = int(os.getenv("HEALTH_CHECK_PORT", "8080"))
+        return value if 1 <= value <= 65535 else 8080
+    except ValueError:
+        return 8080
+
+
+HEALTH_CHECK_PORT = _configured_health_port()
+
+
+def filter_reportable_labels(labels):
+    return {
+        key: value for key, value in (labels or {}).items()
+        if isinstance(key, str)
+        and isinstance(value, str)
+        and key.startswith(("dockflare.", "cloudflare.tunnel."))
+    }
+
+
+def normalize_docker_event_type(action):
+    if action == "start":
+        return "container_start"
+    if action in {"stop", "die"}:
+        return "container_stop"
+    return None
+
+
+def apply_registration_response(data):
+    global AGENT_ID, PROTOCOL_VERSION, AGENT_SESSION_ID, _event_sequence, _report_sequence
+    agent_id = data.get("agent_id") if isinstance(data, dict) else None
+    protocol = data.get("protocol_version") if isinstance(data, dict) else None
+    session_id = data.get("agent_session_id") if isinstance(data, dict) else None
+    if not agent_id:
+        return False
+    if protocol == 2 and not session_id:
+        return False
+    if protocol not in (None, 1, 2):
+        return False
+    with _protocol_lock:
+        AGENT_ID = agent_id
+        PROTOCOL_VERSION = protocol or 1
+        AGENT_SESSION_ID = session_id if protocol == 2 else None
+        _event_sequence = 0
+        _report_sequence = 0
+    return True
+
+
+def next_event_sequence():
+    global _event_sequence
+    with _protocol_lock:
+        _event_sequence += 1
+        return _event_sequence
+
+
+def next_report_sequence():
+    global _report_sequence
+    with _protocol_lock:
+        _report_sequence += 1
+        return _report_sequence
+
+
+def collect_complete_inventory(docker_client):
+    containers = []
+    for container in docker_client.containers.list():
+        labels = getattr(container, "labels", None) or {}
+        if not is_dockflare_enabled(labels):
+            continue
+        containers.append({
+            "id": str(container.id),
+            "name": str(container.name),
+            "labels": filter_reportable_labels(labels),
+            "status": str(getattr(container, "status", "unknown")),
+        })
+    return containers
+
+
+def send_inventory_report(docker_client):
+    """Send one inventory report without ever claiming a failed scan is complete."""
+    try:
+        containers = collect_complete_inventory(docker_client)
+    except Exception as exc:
+        logging.error("Docker inventory scan failed: %s", type(exc).__name__)
+        return send_status_report(inventory_complete=False)
+    return send_status_report(containers, True)
+
+
+def fetch_cloudflared_version(container):
+    try:
+        exec_result = container.exec_run("cloudflared --version")
+        output = getattr(exec_result, 'output', b'') or b''
+        version_text = output.decode('utf-8', errors='ignore').strip()
+        if version_text:
+            first_line = version_text.splitlines()[0]
+            logging.info(f"Detected cloudflared version: {first_line}")
+            return first_line
+    except Exception as e:
+        logging.error(f"Failed to fetch cloudflared version: {e}")
+    return None
+
+
+def load_tunnel_state():
+    global current_tunnel_token, current_tunnel_id, current_tunnel_name, desired_tunnel_state
+    if not os.path.exists(TUNNEL_STATE_FILE):
+        return
+    try:
+        with open(TUNNEL_STATE_FILE, 'r') as f:
+            data = json.load(f)
+        current_tunnel_token = data.get("token")
+        current_tunnel_id = data.get("id")
+        current_tunnel_name = data.get("name")
+        desired_tunnel_state = data.get("desired_state", "unknown")
+        logging.info("Loaded tunnel state from disk.")
+    except Exception as e:
+        logging.error(f"Failed to load tunnel state file: {e}")
+
+
+def save_tunnel_state():
+    data = {
+        "token": current_tunnel_token,
+        "id": current_tunnel_id,
+        "name": current_tunnel_name,
+        "desired_state": desired_tunnel_state
+    }
+    try:
+        _write_secure_file(TUNNEL_STATE_FILE, lambda fh: json.dump(data, fh))
+    except Exception as e:
+        logging.error(f"Failed to persist tunnel state: {e}")
+
+
+def _write_secure_file(path, writer):
+    directory = os.path.dirname(path)
+    os.makedirs(directory, exist_ok=True)
+    temp_path = None
+    try:
+        with tempfile.NamedTemporaryFile('w', dir=directory, delete=False) as tmp_file:
+            writer(tmp_file)
+            tmp_file.flush()
+            try:
+                os.fsync(tmp_file.fileno())
+            except (AttributeError, OSError):
+                pass
+            temp_path = tmp_file.name
+        try:
+            os.chmod(temp_path, 0o600)
+        except OSError:
+            pass
+        os.replace(temp_path, path)
+    except Exception:
+        if temp_path and os.path.exists(temp_path):
+            try:
+                os.remove(temp_path)
+            except OSError:
+                pass
+        raise
+
+
+def ensure_cloudflared_running(client):
+    global tunnel_container, current_tunnel_version
+    if decommission_tombstone or desired_tunnel_state != "running" or not current_tunnel_token or not current_tunnel_name:
+        return
+    try:
+        existing = client.containers.get('dockflare-agent-tunnel')
+        existing.reload()
+        if existing.status == 'running':
+            tunnel_container = existing
+            if not current_tunnel_version:
+                current_tunnel_version = fetch_cloudflared_version(existing)
+            return
+        logging.warning(f"Detected cloudflared container in status '{existing.status}'. Redeploying.")
+    except docker.errors.NotFound:
+        logging.warning("cloudflared container missing. Redeploying.")
+    except Exception as e:
+        logging.error(f"Error inspecting cloudflared container: {e}")
+
+    _run_cloudflared_container(client, current_tunnel_name, current_tunnel_token)
+
+
+def _remove_existing_container(client):
+    global tunnel_container, current_tunnel_version
+    try:
+        existing_container = client.containers.get('dockflare-agent-tunnel')
+        logging.info(f"Stopping existing cloudflared container '{existing_container.name}' ({existing_container.short_id}).")
+        existing_container.stop()
+        existing_container.remove()
+        if tunnel_container and tunnel_container.id == existing_container.id:
+            tunnel_container = None
+    except docker.errors.NotFound:
+        logging.info("No existing cloudflared container to remove.")
+    except Exception as e:
+        logging.error(f"Failed to remove existing cloudflared container: {e}")
+    finally:
+        current_tunnel_version = None
+
+
+def _run_cloudflared_container(client, tunnel_name, tunnel_token):
+    global tunnel_container, current_tunnel_version
+    if not tunnel_token:
+        logging.error("Cannot start cloudflared: missing tunnel token.")
+        return False
+    try:
+        tunnel_container = client.containers.run(
+            CLOUDFLARED_IMAGE,
+            command=["tunnel", "--no-autoupdate", "run"],
+            detach=True,
+            name="dockflare-agent-tunnel",
+            network=CLOUDFLARED_NETWORK_NAME,
+            restart_policy={"Name": "unless-stopped"},
+            environment={"TUNNEL_TOKEN": tunnel_token}
+        )
+        time.sleep(2)
+        current_tunnel_version = fetch_cloudflared_version(tunnel_container)
+        logging.info(f"cloudflared container started: {tunnel_container.short_id}")
+        report_event_to_master("tunnel_status", {
+            "name": tunnel_name,
+            "status": "running",
+            "version": current_tunnel_version
+        })
+        return True
+    except Exception as e:
+        logging.error(f"Failed to start cloudflared container: {e}")
+        return False
+
+def load_agent_id():
+    """Loads agent ID from the filesystem."""
+    global AGENT_ID
+    if os.path.exists(AGENT_ID_FILE):
+        try:
+            with open(AGENT_ID_FILE, 'r') as f:
+                agent_id = f.read().strip()
+                if agent_id:
+                    AGENT_ID = agent_id
+                    logging.info(f"Loaded existing Agent ID: {AGENT_ID}")
+        except IOError as e:
+            logging.error(f"Could not read agent ID file: {e}")
+
+def save_agent_id(agent_id):
+    """Saves agent ID to the filesystem."""
+    try:
+        _write_secure_file(AGENT_ID_FILE, lambda fh: fh.write(agent_id))
+        logging.info(f"Saved Agent ID to {AGENT_ID_FILE}")
+    except IOError as e:
+        logging.error(f"Could not save agent ID file: {e}")
+
+def register_with_master():
+    """
+    Registers the agent with the DockFlare Master instance and retrieves an agent ID.
+    """
+    global AGENT_ID, last_successful_master_contact
+    if not MASTER_URL or not API_KEY:
+        logging.error("Error: DOCKFLARE_MASTER_URL and DOCKFLARE_API_KEY must be set.")
+        return False
+
+    endpoint = f"{MASTER_URL}/api/v2/agents/register"
+    while True:
+        logging.info(f"Attempting to register with master at {endpoint}")
+        try:
+            headers = get_headers()
+            custom_display_name = os.getenv("AGENT_DISPLAY_NAME", "").strip()
+            default_display_name = f"agent-{AGENT_ID[:8]}" if AGENT_ID else "dockflare-agent"
+            payload = {
+                "display_name": custom_display_name or default_display_name,
+                "version": AGENT_VERSION,
+                "agent_version": AGENT_VERSION,
+                "supported_protocol_versions": [2, 1],
+                "capabilities": ["decommission.v1", "tunnel_stop.v1", "self_stop.v1"],
+            }
+            if AGENT_ID:
+                payload["agent_id"] = AGENT_ID
+
+            response = requests.post(endpoint, json=payload, headers=headers, timeout=15)
+            response.raise_for_status()
+            data = response.json()
+            new_agent_id = data.get("agent_id")
+            if not apply_registration_response(data):
+                logging.error("Master did not provide an agent_id. Retrying in 60s...")
+                time.sleep(60)
+                continue
+
+            if AGENT_ID and AGENT_ID != new_agent_id:
+                logging.warning(f"Master assigned a new Agent ID ({new_agent_id}). Overwriting old one ({AGENT_ID}).")
+
+            AGENT_ID = new_agent_id
+            save_agent_id(AGENT_ID)
+            last_successful_master_contact = datetime.now(timezone.utc)
+            logging.info(f"Successfully registered with master. Agent ID: {AGENT_ID}")
+            return True
+        except requests.exceptions.RequestException as e:
+            logging.error(f"Error registering with master: {e}. Retrying in 60s...")
+            time.sleep(60)
+
+def _safe_response_code(response):
+    try:
+        data = response.json()
+        return str(data.get("code") or "unknown")[:64]
+    except (ValueError, AttributeError):
+        return "invalid_response"
+
+
+def _post_agent_payload(payload):
+    global last_successful_master_contact
+    endpoint = f"{MASTER_URL}/api/v2/agents/{AGENT_ID}/events"
+    for attempt in range(3):
+        try:
+            response = requests.post(endpoint, json=payload, headers=get_headers(), timeout=15)
+            code = _safe_response_code(response)
+            accepted_code = code in {"accepted", "accepted_noop", "duplicate_sequence", "inventory_incomplete"}
+            if response.status_code in (200, 202) and (accepted_code or PROTOCOL_VERSION == 1):
+                last_successful_master_contact = datetime.now(timezone.utc)
+                return True
+            if response.status_code == 409 and code == "registration_required":
+                with _registration_lock:
+                    if register_with_master():
+                        return False
+            if response.status_code == 429 or response.status_code >= 500:
+                if attempt < 2:
+                    retry_after = response.headers.get("Retry-After") if hasattr(response, "headers") else None
+                    try:
+                        retry_delay = min(60.0, max(0.0, float(retry_after)))
+                    except (TypeError, ValueError):
+                        retry_delay = (2 ** attempt) + random.random()
+                    time.sleep(retry_delay)
+                    continue
+            logging.error("Master rejected Agent request (status=%s, code=%s)", response.status_code, code)
+            return False
+        except requests.exceptions.RequestException as exc:
+            if attempt == 2:
+                logging.error("Agent request failed after bounded retries: %s", type(exc).__name__)
+                return False
+            time.sleep((2 ** attempt) + random.random())
+    return False
+
+
+def report_event_to_master(event_type, container_data=None):
+    """
+    Sends a JSON payload to the master's reporting endpoint.
+    """
+    if not AGENT_ID or decommission_tombstone:
+        logging.debug("report_event_to_master called but AGENT_ID is missing; skipping.")
+        return
+    try:
+        payload = {
+            "type": event_type,
+            "timestamp": datetime.now(timezone.utc).isoformat()
+        }
+        if container_data:
+            safe_container = dict(container_data)
+            safe_container["labels"] = filter_reportable_labels(safe_container.get("labels", {}))
+            payload["container"] = safe_container
+        with _event_send_lock:
+            if PROTOCOL_VERSION == 2:
+                payload.update({
+                    "protocol_version": 2,
+                    "agent_session_id": AGENT_SESSION_ID,
+                    "event_sequence": next_event_sequence(),
+                })
+            return _post_agent_payload(payload)
+    except Exception as exc:
+        logging.error("Could not build Agent event: %s", type(exc).__name__)
+        return False
+
+
+def send_status_report(containers=None, inventory_complete=True):
+    if not AGENT_ID or decommission_tombstone:
+        return False
+    with _report_send_lock:
+        if PROTOCOL_VERSION != 2:
+            return report_event_to_master("status_report", {"containers": containers or []})
+        payload = {
+            "type": "status_report",
+            "protocol_version": 2,
+            "agent_session_id": AGENT_SESSION_ID,
+            "report_sequence": next_report_sequence(),
+            "inventory_complete": bool(inventory_complete),
+            "inventory_scope": "dockflare_enabled_running",
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+        }
+        if inventory_complete:
+            payload["containers"] = containers or []
+        return _post_agent_payload(payload)
+
+
+def get_docker_event_container_id(event):
+    if not isinstance(event, dict):
+        return None
+
+    actor = event.get("Actor")
+    actor_id = actor.get("ID") if isinstance(actor, dict) else None
+    for value in (actor_id, event.get("id"), event.get("ID")):
+        if isinstance(value, str) and value.strip():
+            return value.strip()
+    return None
+
+
+def listen_for_docker_events(client, label_prefix, initial_scan=True):
+    thread_health_status[f"events_listener_{label_prefix.strip('.')}"] = "running"
+    if initial_scan:
+        logging.info("Performing one initial scan of running DockFlare containers...")
+        try:
+            for container in client.containers.list():
+                if is_dockflare_enabled(container.labels):
+                    report_event_to_master("container_start", {
+                        "id": container.id,
+                        "name": container.name,
+                        "labels": container.labels,
+                    })
+        except Exception as e:
+            logging.error(f"Error during initial container scan: {e}")
+
+    logging.info(f"Listening for Docker events with prefix {label_prefix}...")
+    event_filters = {
+        "type": "container",
+        "label": f"{label_prefix}enable=true"
+    }
+    try:
+        for event in client.events(decode=True, filters=event_filters):
+            try:
+                if event.get("Type") == "container" and event.get("Action") in ["start", "stop", "die"]:
+                    action = event["Action"]
+                    container_id = get_docker_event_container_id(event)
+                    if not container_id:
+                        logging.warning(
+                            "Ignoring Docker '%s' event without a container identifier (%s)",
+                            action,
+                            label_prefix,
+                        )
+                        continue
+                    try:
+                        container = client.containers.get(container_id)
+                        event_type = normalize_docker_event_type(action)
+                        logging.info(f"Detected event '{action}' for container {container.name} ({label_prefix})")
+                        report_event_to_master(event_type, {
+                            "id": container_id,
+                            "name": container.name,
+                            "labels": container.labels
+                        })
+                    except docker.errors.NotFound:
+                        logging.warning(f"Container {container_id[:12]} not found after event '{action}' ({label_prefix}). Reporting to master.")
+                        event_type = normalize_docker_event_type(action)
+                        actor = event.get("Actor", {}).get("Attributes", {}) or {}
+                        report_event_to_master(event_type, {
+                            "id": container_id,
+                            "name": actor.get("name", "unknown"),
+                            "labels": filter_reportable_labels(actor),
+                        })
+            except Exception as e:
+                logging.error(f"An error occurred in the event loop for {label_prefix}: {e}")
+    except Exception as e:
+        logging.error(f"Failed to connect to Docker event stream for {label_prefix}: {e}")
+        thread_health_status[f"events_listener_{label_prefix.strip('.')}"] = "failed"
+
+def start_event_listeners(client):
+    threads = []
+    label_prefixes = ["dockflare.", "cloudflare.tunnel."]
+    for index, prefix in enumerate(label_prefixes):
+        thread = Thread(target=listen_for_docker_events, args=(client, prefix, index == 0), daemon=True)
+        threads.append(thread)
+    return threads
+
+
+def post_decommission_ack(operation_id, payload):
+    global last_successful_master_contact
+    endpoint = f"{MASTER_URL}/api/v2/agents/{AGENT_ID}/decommission/{operation_id}/ack"
+    body = dict(payload)
+    body["operation_id"] = operation_id
+    if PROTOCOL_VERSION == 2:
+        body["agent_session_id"] = AGENT_SESSION_ID
+    try:
+        response = requests.post(endpoint, json=body, headers=get_headers(), timeout=60)
+        if response.status_code == 200 and _safe_response_code(response) == "acknowledged":
+            last_successful_master_contact = datetime.now(timezone.utc)
+            return True
+        logging.error(
+            "Master rejected decommission acknowledgement (status=%s, code=%s)",
+            response.status_code,
+            _safe_response_code(response),
+        )
+    except requests.exceptions.RequestException as exc:
+        logging.error("Decommission acknowledgement failed: %s", type(exc).__name__)
+    return False
+
+
+def prepare_decommission(client, command):
+    global desired_tunnel_state, decommission_tombstone
+    operation_id = command.get("operation_id")
+    command_id = command.get("command_id")
+    if not operation_id or not command_id or command.get("requested_self_action") != "stop":
+        return False
+    try:
+        decommission_tombstone = decommission_runtime.persist_tombstone(operation_id, AGENT_ID, "prepared")
+        desired_tunnel_state = "stopped"
+        save_tunnel_state()
+    except Exception:
+        logging.error("Could not persist decommission preparation state.")
+        return post_decommission_ack(operation_id, {
+            "command_id": command_id,
+            "phase": "prepared",
+            "tombstone_persisted": False,
+            "tunnel_container": "failed",
+            "error_code": "tombstone_persistence_failed",
+        })
+
+    tunnel_result, tunnel_image = decommission_runtime.stop_tunnel_container(client)
+    self_image = decommission_runtime.self_image_reference(client)
+    payload = {
+        "command_id": command_id,
+        "phase": "prepared",
+        "tombstone_persisted": True,
+        "tunnel_container": tunnel_result,
+        "self_stop_capability": "supported" if self_image else "manual",
+        "agent_image": self_image,
+        "cloudflared_image": tunnel_image or CLOUDFLARED_IMAGE,
+    }
+    if tunnel_result == "failed":
+        payload["error_code"] = "tunnel_stop_failed"
+    return post_decommission_ack(operation_id, payload)
+
+
+def finalize_decommission(client, command):
+    global current_tunnel_token, current_tunnel_id, current_tunnel_name
+    global desired_tunnel_state, decommission_tombstone
+    operation_id = command.get("operation_id")
+    command_id = command.get("command_id")
+    if not operation_id or not command_id:
+        return False
+    try:
+        decommission_tombstone = decommission_runtime.persist_tombstone(operation_id, AGENT_ID, "finalized")
+        current_tunnel_token = None
+        current_tunnel_id = None
+        current_tunnel_name = None
+        desired_tunnel_state = "stopped"
+        save_tunnel_state()
+    except Exception:
+        logging.error("Could not persist final decommission state.")
+        return False
+    self_stop_supported = decommission_runtime.resolve_self_container(client) is not None
+    acknowledged = post_decommission_ack(operation_id, {
+        "command_id": command_id,
+        "phase": "shutdown_scheduled",
+        "self_stop_capability": "supported" if self_stop_supported else "manual",
+    })
+    if acknowledged and self_stop_supported:
+        return decommission_runtime.schedule_self_stop(client)
+    return acknowledged
+
+def manage_tunnels(client):
+    """
+    Periodically polls for commands from the master and manages a cloudflared container.
+    """
+    global tunnel_container, current_tunnel_token, current_tunnel_id, current_tunnel_version, current_tunnel_name, desired_tunnel_state, last_successful_master_contact
+    logging.info("Tunnel management thread started.")
+
+    while True:
+        if decommission_tombstone and decommission_tombstone.get("phase") == "finalized":
+            logging.info("Agent decommission is finalized; command polling is stopped.")
+            return
+        if not AGENT_ID:
+            time.sleep(10)
+            continue
+        try:
+            headers = get_headers(content_type=False)
+            endpoint = f"{MASTER_URL}/api/v2/agents/{AGENT_ID}/commands"
+            response = requests.get(endpoint, headers=headers, timeout=15)
+            response.raise_for_status()
+            last_successful_master_contact = datetime.now(timezone.utc)
+            commands = response.json().get("commands", [])
+
+            for cmd in commands:
+                action = cmd.get("action")
+                if action == "prepare_decommission":
+                    prepare_decommission(client, cmd)
+
+                elif action == "finalize_decommission":
+                    finalize_decommission(client, cmd)
+
+                elif decommission_tombstone:
+                    logging.warning("Ignoring tunnel command while Agent decommission is active.")
+
+                elif action == "start_tunnel":
+                    tunnel_token = cmd.get("token")
+                    tunnel_name = cmd.get("tunnel_name")
+                    tunnel_id = cmd.get("tunnel_id")
+                    token_changed = tunnel_token and tunnel_token != current_tunnel_token
+                    id_changed = tunnel_id and tunnel_id != current_tunnel_id
+                    if token_changed or id_changed:
+                        reason_parts = []
+                        if token_changed:
+                            reason_parts.append("token change")
+                        if id_changed:
+                            reason_parts.append("tunnel id change")
+                        reason_text = ", ".join(reason_parts) if reason_parts else "assignment update"
+                        logging.info(f"Received command to start tunnel '{tunnel_name}' ({reason_text}).")
+
+                        _remove_existing_container(client)
+                        logging.info("Starting new cloudflared tunnel container...")
+                        _run_cloudflared_container(client, tunnel_name, tunnel_token)
+                        current_tunnel_token = tunnel_token
+                        current_tunnel_id = tunnel_id
+                        current_tunnel_name = tunnel_name
+                        desired_tunnel_state = "running"
+                        save_tunnel_state()
+                    else:
+                        logging.info("Received start_tunnel command but tunnel token/id unchanged; ensuring cloudflared is running.")
+                        current_tunnel_name = tunnel_name or current_tunnel_name
+                        desired_tunnel_state = "running"
+                        save_tunnel_state()
+                        ensure_cloudflared_running(client)
+
+                elif action == "restart_tunnel":
+                    tunnel_token = cmd.get("tunnel_token")
+                    tunnel_name = cmd.get("tunnel_name")
+                    tunnel_id = cmd.get("tunnel_id")
+                    if tunnel_token and tunnel_name and tunnel_id:
+                        logging.info(f"Received command to restart tunnel '{tunnel_name}'.")
+                        _remove_existing_container(client)
+                        logging.info("Restarting cloudflared tunnel container...")
+                        _run_cloudflared_container(client, tunnel_name, tunnel_token)
+                        current_tunnel_token = tunnel_token
+                        current_tunnel_id = tunnel_id
+                        current_tunnel_name = tunnel_name
+                        desired_tunnel_state = "running"
+                        save_tunnel_state()
+                    else:
+                        logging.warning("Received restart_tunnel command with missing parameters.")
+
+                elif action == "stop_tunnel":
+                    logging.info("Received command to stop cloudflared tunnel container.")
+                    desired_tunnel_state = "stopped"
+                    save_tunnel_state()
+                    _remove_existing_container(client)
+                    report_event_to_master("tunnel_status", {
+                        "name": current_tunnel_name,
+                        "status": "stopped"
+                    })
+                    current_tunnel_token = None
+                    current_tunnel_id = None
+                    current_tunnel_name = None
+                    save_tunnel_state()
+
+                elif action == "update_tunnel_config":
+                    if not current_tunnel_id:
+                        logging.error("Cannot update tunnel config: no tunnel ID available.")
+                        continue
+                    rules = cmd.get("rules", {})
+                    ingress_rules = cloudflare_api.generate_ingress_rules(rules)
+                    success = cloudflare_api.update_tunnel_config(MASTER_URL, API_KEY, current_tunnel_id, ingress_rules)
+                    if success:
+                        logging.info("Tunnel configuration updated successfully")
+                    else:
+                        logging.error("Failed to update tunnel configuration")
+
+        except requests.exceptions.RequestException as e:
+            logging.error(f"Could not poll commands from master: {e}")
+        except docker.errors.ImageNotFound:
+            logging.error("Could not start tunnel: cloudflare/cloudflared image not found. Please pull it.")
+        except Exception as e:
+            logging.error(f"An error in tunnel management: {e}")
+        time.sleep(30)  # Poll every 30 seconds
+
+
+def tunnel_health_monitor(client):
+    logging.info("Tunnel health monitor thread started.")
+    while True:
+        try:
+            ensure_cloudflared_running(client)
+        except Exception as e:
+            logging.error(f"Tunnel health monitor encountered an error: {e}")
+        time.sleep(30)
+
+REPORT_INTERVAL_SECONDS = int(os.getenv("REPORT_INTERVAL_SECONDS", "30"))
+
+def periodic_status_reporter(client):
+    """
+    Periodic reporter: sends heartbeat and full status_report of enabled containers.
+    """
+    logging.info("Status reporter thread started.")
+    while True:
+        if decommission_tombstone:
+            time.sleep(REPORT_INTERVAL_SECONDS)
+            continue
+        if not AGENT_ID:
+            time.sleep(5)
+            continue
+        try:
+
+            logging.info("Sending heartbeat to master.")
+            report_event_to_master("heartbeat")
+
+            send_inventory_report(client)
+        except Exception as e:
+            logging.error(f"Periodic reporter error: {e}")
+        time.sleep(REPORT_INTERVAL_SECONDS)
+
+def cleanup():
+    """
+    Gracefully shuts down the agent and its resources.
+    """
+    global tunnel_container, current_tunnel_version
+    logging.info("Shutting down agent.")
+    if tunnel_container:
+        logging.info("Stopping tunnel container...")
+        try:
+            tunnel_container.stop()
+            tunnel_container.remove()
+        except docker.errors.NotFound:
+            pass
+        except Exception as e:
+            logging.error(f"Could not stop tunnel container: {e}")
+        finally:
+            tunnel_container = None
+    current_tunnel_version = None
+# --- Health Check HTTP Server ---
+class HealthCheckHandler(http.server.SimpleHTTPRequestHandler):
+    def do_GET(self):
+        global last_successful_master_contact, thread_health_status
+
+        if self.path == '/health':
+            seconds_since_contact = None
+            if last_successful_master_contact:
+                seconds_since_contact = int((datetime.now(timezone.utc) - last_successful_master_contact).total_seconds())
+
+            lifecycle = decommission_tombstone.get("phase") if decommission_tombstone else "active"
+            status = "decommissioned" if lifecycle == "finalized" else "prepared" if lifecycle == "prepared" else "unhealthy"
+            if AGENT_ID and not decommission_tombstone:
+                if seconds_since_contact is not None and seconds_since_contact < 120:
+                    status = "healthy"
+                elif seconds_since_contact is not None and seconds_since_contact < 300:
+                    status = "degraded"
+
+            tunnel_container_running = False
+            if tunnel_container:
+                try:
+                    tunnel_container.reload()
+                    tunnel_container_running = tunnel_container.status == 'running'
+                except:
+                    pass
+
+            response_data = {
+                "status": status,
+                "agent_id": AGENT_ID,
+                "registered": AGENT_ID is not None,
+                "lifecycle": lifecycle,
+                "tunnel": {
+                    "state": desired_tunnel_state,
+                    "name": current_tunnel_name,
+                    "id": current_tunnel_id,
+                    "container_running": tunnel_container_running,
+                    "version": current_tunnel_version
+                },
+                "master_connection": {
+                    "url": MASTER_URL,
+                    "last_successful_report": last_successful_master_contact.isoformat() if last_successful_master_contact else None,
+                    "seconds_since_contact": seconds_since_contact
+                },
+                "threads": thread_health_status.copy()
+            }
+
+            http_status = 200 if status in {"healthy", "prepared", "decommissioned"} else 503
+            self.send_response(http_status)
+            self.send_header('Content-type', 'application/json')
+            self.end_headers()
+            self.wfile.write(json.dumps(response_data).encode('utf-8'))
+        else:
+            self.send_response(404)
+            self.end_headers()
+            self.wfile.write(b'404 Not Found')
+
+def run_health_check_server():
+    thread_health_status["health_server"] = "running"
+    try:
+        with socketserver.TCPServer(('', HEALTH_CHECK_PORT), HealthCheckHandler) as httpd:
+            logging.info(f"Health check server running on port {HEALTH_CHECK_PORT}")
+            httpd.serve_forever()
+    except Exception as e:
+        logging.error(f"Health check server error: {e}")
+        thread_health_status["health_server"] = "failed"
+
+if __name__ == "__main__":
+    decommission_tombstone = decommission_runtime.load_tombstone()
+    load_agent_id()
+    load_tunnel_state()
+    if decommission_tombstone and decommission_tombstone.get("phase") == "finalized":
+        logging.warning("Agent is finalized and will remain dormant until its deployment is removed or reset.")
+        Thread(target=run_health_check_server, daemon=True).start()
+        while True:
+            time.sleep(60)
+    elif register_with_master():
+        docker_client = docker.from_env()
+        try:
+            if not decommission_tombstone:
+                ensure_cloudflared_running(docker_client)
+            tunnel_thread = Thread(target=manage_tunnels, args=(docker_client,), daemon=True)
+            tunnel_thread.start()
+            if not decommission_tombstone:
+                status_thread = Thread(target=periodic_status_reporter, args=(docker_client,), daemon=True)
+                status_thread.start()
+                event_threads = start_event_listeners(docker_client)
+                for t in event_threads:
+                    t.start()
+                monitor_thread = Thread(target=tunnel_health_monitor, args=(docker_client,), daemon=True)
+                monitor_thread.start()
+            health_thread = Thread(target=run_health_check_server, daemon=True)
+            health_thread.start()
+            while True:
+                time.sleep(1)
+
+        except KeyboardInterrupt:
+            cleanup()
+        except Exception as e:
+            logging.critical(f"A critical error occurred: {e}")
+            cleanup()
+    else:
+        logging.critical("Agent startup failed. Could not register with master.")

dockflare-agent/docker-compose.yml

@@ -0,0 +1,49 @@
+services:
+  docker-socket-proxy:
+    image: tecnativa/docker-socket-proxy:v0.4.1
+    container_name: dockflare-socket-proxy
+    restart: unless-stopped
+    environment:
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - CONTAINERS=1
+      - EVENTS=1
+      - NETWORKS=1
+      - IMAGES=1
+      - POST=1
+      - PING=1
+      - EXEC=1
+      - DELETE=0
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    networks:
+      - dockflare-internal
+
+  dockflare-agent:
+    build: .
+    container_name: dockflare-agent
+    restart: unless-stopped
+    env_file:
+      - .env
+    environment:
+      - DOCKER_HOST=${DOCKER_HOST:-tcp://docker-socket-proxy:2375}
+      - TZ=${TZ:-UTC}
+      - LOG_LEVEL=${LOG_LEVEL:-info}
+      - CF_ACCESS_CLIENT_ID=${CF_ACCESS_CLIENT_ID:-}
+      - CF_ACCESS_CLIENT_SECRET=${CF_ACCESS_CLIENT_SECRET:-}
+    volumes:
+      - agent_data:/app/data
+    depends_on:
+      - docker-socket-proxy
+    networks:
+      - cloudflare-net
+      - dockflare-internal
+
+volumes:
+  agent_data:
+
+networks:
+  cloudflare-net:
+    name: cloudflare-net
+    external: true
+  dockflare-internal:
+    name: dockflare-internal

dockflare-agent/env-example

@@ -0,0 +1,10 @@
+DOCKFLARE_MASTER_URL=https://dockflare.example.com
+DOCKFLARE_API_KEY=agent_api_key_goes_here
+DOCKER_HOST=tcp://docker-socket-proxy:2375
+AGENT_DISPLAY_NAME=Production Server
+CLOUDFLARED_IMAGE=cloudflare/cloudflared:2025.9.0
+LOG_LEVEL=info
+HEALTH_CHECK_PORT=8080
+TZ=Europe/Zurich
+CF_ACCESS_CLIENT_ID=
+CF_ACCESS_CLIENT_SECRET=

dockflare-agent/requirements.txt

@@ -0,0 +1,3 @@
+requests==2.33.1
+docker==7.1.0
+python-dotenv==1.2.2

dockflare-agent/tests/test_agent_protocol.py

@@ -0,0 +1,218 @@
+import importlib.util
+import pathlib
+import sys
+import unittest
+from unittest.mock import patch
+
+
+AGENT_ROOT = pathlib.Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(AGENT_ROOT / "app"))
+SPEC = importlib.util.spec_from_file_location("dockflare_agent_main", AGENT_ROOT / "app" / "main.py")
+agent = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(agent)
+
+
+class FakeContainer:
+    def __init__(self, container_id, name, labels, status="running"):
+        self.id = container_id
+        self.name = name
+        self.labels = labels
+        self.status = status
+
+
+class FakeContainers:
+    def __init__(self, values, missing_ids=None):
+        self.values = values
+        self.by_id = {container.id: container for container in values}
+        self.missing_ids = set(missing_ids or [])
+
+    def list(self):
+        return list(self.values)
+
+    def get(self, container_id):
+        if container_id in self.missing_ids or container_id not in self.by_id:
+            raise agent.docker.errors.NotFound("container not found")
+        return self.by_id[container_id]
+
+
+class FakeDocker:
+    def __init__(self, values, events=None, missing_ids=None):
+        self.containers = FakeContainers(values, missing_ids)
+        self.event_values = list(events or [])
+
+    def events(self, **_kwargs):
+        return iter(self.event_values)
+
+
+class AgentProducerProtocolTests(unittest.TestCase):
+    def setUp(self):
+        agent.AGENT_ID = None
+        agent.PROTOCOL_VERSION = None
+        agent.AGENT_SESSION_ID = None
+        agent._event_sequence = 0
+        agent._report_sequence = 0
+        agent.decommission_tombstone = None
+        agent.desired_tunnel_state = "unknown"
+
+    def test_registration_context_requires_v2_session_and_resets_sequences(self):
+        self.assertFalse(agent.apply_registration_response({
+            "agent_id": "agent-a", "protocol_version": 2,
+        }))
+        agent._event_sequence = 9
+        self.assertTrue(agent.apply_registration_response({
+            "agent_id": "agent-a", "protocol_version": 2, "agent_session_id": "master-session",
+        }))
+        self.assertEqual(agent.next_event_sequence(), 1)
+        self.assertEqual(agent.next_report_sequence(), 1)
+
+    def test_inventory_is_complete_filtered_and_deduplicated_by_scan(self):
+        docker_client = FakeDocker([
+            FakeContainer("one", "one", {
+                "dockflare.enable": "true",
+                "dockflare.hostname": "one.example.com",
+                "compose.project": "private",
+            }),
+            FakeContainer("two", "two", {"app.enabled": "true"}),
+        ])
+        inventory = agent.collect_complete_inventory(docker_client)
+        self.assertEqual(len(inventory), 1)
+        self.assertEqual(inventory[0]["labels"], {
+            "dockflare.enable": "true",
+            "dockflare.hostname": "one.example.com",
+        })
+
+    def test_inventory_scan_failure_sends_explicit_incomplete_report(self):
+        with patch.object(agent, "collect_complete_inventory", side_effect=RuntimeError("docker unavailable")), \
+             patch.object(agent, "send_status_report", return_value=True) as send_report:
+            self.assertTrue(agent.send_inventory_report(object()))
+
+        send_report.assert_called_once_with(inventory_complete=False)
+
+    def test_status_report_uses_top_level_v2_shape(self):
+        agent.apply_registration_response({
+            "agent_id": "agent-a", "protocol_version": 2, "agent_session_id": "master-session",
+        })
+        captured = []
+        with patch.object(agent, "_post_agent_payload", side_effect=lambda payload: captured.append(payload) or True):
+            self.assertTrue(agent.send_status_report([], True))
+        self.assertEqual(captured[0]["containers"], [])
+        self.assertEqual(captured[0]["inventory_scope"], "dockflare_enabled_running")
+        self.assertTrue(captured[0]["inventory_complete"])
+        self.assertNotIn("container", captured[0])
+
+    def test_event_filters_labels_and_allocates_sequence(self):
+        agent.apply_registration_response({
+            "agent_id": "agent-a", "protocol_version": 2, "agent_session_id": "master-session",
+        })
+        captured = []
+        with patch.object(agent, "_post_agent_payload", side_effect=lambda payload: captured.append(payload) or True):
+            agent.report_event_to_master("container_start", {
+                "id": "container", "name": "service",
+                "labels": {"dockflare.enable": "true", "secret": "DO_NOT_SEND"},
+            })
+        self.assertEqual(captured[0]["event_sequence"], 1)
+        self.assertNotIn("secret", captured[0]["container"]["labels"])
+
+    def test_docker_event_types_are_normalized(self):
+        self.assertEqual(agent.normalize_docker_event_type("start"), "container_start")
+        self.assertEqual(agent.normalize_docker_event_type("stop"), "container_stop")
+        self.assertEqual(agent.normalize_docker_event_type("die"), "container_stop")
+        self.assertIsNone(agent.normalize_docker_event_type("destroy"))
+
+    def test_event_container_id_prefers_canonical_actor_shape(self):
+        event = {
+            "Actor": {"ID": "actor-id"},
+            "id": "legacy-id",
+        }
+        self.assertEqual(agent.get_docker_event_container_id(event), "actor-id")
+        self.assertEqual(agent.get_docker_event_container_id({"id": "legacy-id"}), "legacy-id")
+        self.assertIsNone(agent.get_docker_event_container_id({"Actor": {"Attributes": {}}}))
+
+    def test_start_event_uses_actor_id(self):
+        container = FakeContainer("container-id", "service", {
+            "dockflare.enable": "true",
+            "dockflare.hostname": "service.example.com",
+        })
+        client = FakeDocker([container], events=[{
+            "Type": "container",
+            "Action": "start",
+            "Actor": {"ID": "container-id", "Attributes": {"name": "service"}},
+        }])
+
+        with patch.object(agent, "report_event_to_master") as report:
+            agent.listen_for_docker_events(client, "dockflare.", initial_scan=False)
+
+        report.assert_called_once_with("container_start", {
+            "id": "container-id",
+            "name": "service",
+            "labels": container.labels,
+        })
+
+    def test_stop_and_die_not_found_events_use_actor_fallback(self):
+        for action in ("stop", "die"):
+            with self.subTest(action=action):
+                client = FakeDocker([], missing_ids={"removed-id"}, events=[{
+                    "Type": "container",
+                    "Action": action,
+                    "Actor": {
+                        "ID": "removed-id",
+                        "Attributes": {
+                            "name": "removed-service",
+                            "dockflare.enable": "true",
+                            "private.label": "do-not-send",
+                        },
+                    },
+                }])
+
+                with patch.object(agent, "report_event_to_master") as report:
+                    agent.listen_for_docker_events(client, "dockflare.", initial_scan=False)
+
+                report.assert_called_once_with("container_stop", {
+                    "id": "removed-id",
+                    "name": "removed-service",
+                    "labels": {"dockflare.enable": "true"},
+                })
+
+    def test_missing_event_id_is_ignored_without_payload_logging(self):
+        client = FakeDocker([], events=[{
+            "Type": "container",
+            "Action": "start",
+            "Actor": {"Attributes": {"name": "service", "secret": "do-not-log"}},
+        }])
+
+        with patch.object(agent, "report_event_to_master") as report, \
+             patch.object(agent.logging, "warning") as warning:
+            agent.listen_for_docker_events(client, "dockflare.", initial_scan=False)
+
+        report.assert_not_called()
+        warning.assert_called_once()
+        self.assertNotIn("secret", " ".join(str(value) for value in warning.call_args.args))
+
+    def test_prepare_persists_tombstone_and_desired_stop_before_docker_stop(self):
+        agent.AGENT_ID = "agent-a"
+        order = []
+        command = {"operation_id": "operation-a", "command_id": "command-a", "requested_self_action": "stop"}
+        with patch.object(agent.decommission_runtime, "persist_tombstone", side_effect=lambda *_args: order.append("tombstone") or {"phase": "prepared"}), \
+             patch.object(agent, "save_tunnel_state", side_effect=lambda: order.append("desired_stop")), \
+             patch.object(agent.decommission_runtime, "stop_tunnel_container", side_effect=lambda _client: order.append("docker_stop") or ("stopped", "cloudflare/cloudflared:latest")), \
+             patch.object(agent.decommission_runtime, "self_image_reference", return_value="alplat/dockflare-agent:dev"), \
+             patch.object(agent, "post_decommission_ack", side_effect=lambda *_args: order.append("ack") or True):
+            self.assertTrue(agent.prepare_decommission(object(), command))
+        self.assertEqual(order, ["tombstone", "desired_stop", "docker_stop", "ack"])
+        self.assertEqual(agent.desired_tunnel_state, "stopped")
+
+    def test_finalize_schedules_self_stop_only_after_ack_success(self):
+        agent.AGENT_ID = "agent-a"
+        command = {"operation_id": "operation-a", "command_id": "command-a"}
+        order = []
+        with patch.object(agent.decommission_runtime, "persist_tombstone", return_value={"phase": "finalized"}), \
+             patch.object(agent, "save_tunnel_state"), \
+             patch.object(agent.decommission_runtime, "resolve_self_container", return_value=object()), \
+             patch.object(agent, "post_decommission_ack", side_effect=lambda *_args: order.append("ack") or True), \
+             patch.object(agent.decommission_runtime, "schedule_self_stop", side_effect=lambda _client: order.append("stop") or True):
+            self.assertTrue(agent.finalize_decommission(object(), command))
+        self.assertEqual(order, ["ack", "stop"])
+
+
+if __name__ == "__main__":
+    unittest.main()

dockflare-agent/tests/test_decommission.py

@@ -0,0 +1,123 @@
+import importlib.util
+import os
+import pathlib
+import stat
+import sys
+import tempfile
+import time
+import unittest
+from unittest.mock import patch
+
+
+AGENT_ROOT = pathlib.Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(AGENT_ROOT / "app"))
+SPEC = importlib.util.spec_from_file_location("dockflare_agent_decommission", AGENT_ROOT / "app" / "decommission.py")
+decommission = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(decommission)
+
+
+class FakeContainer:
+    def __init__(self, container_id, name, status="running", service=None, image="image:tag"):
+        self.id = container_id
+        self.name = name
+        self.status = status
+        self.labels = {"com.docker.compose.service": service} if service else {}
+        self.attrs = {"Config": {"Image": image}}
+        self.stop_calls = 0
+
+    def reload(self):
+        return None
+
+    def stop(self, timeout=15):
+        self.stop_calls += 1
+        self.status = "exited"
+
+
+class FakeContainers:
+    def __init__(self, containers):
+        self.by_name = {container.name: container for container in containers}
+        self.by_id = {container.id: container for container in containers}
+
+    def get(self, value):
+        container = self.by_name.get(value) or self.by_id.get(value)
+        if container is None:
+            matches = [item for container_id, item in self.by_id.items() if container_id.startswith(value)]
+            container = matches[0] if len(matches) == 1 else None
+        if container is None:
+            raise decommission.docker.errors.NotFound("missing")
+        return container
+
+
+class FakeDocker:
+    def __init__(self, containers):
+        self.containers = FakeContainers(containers)
+
+
+class LocalDecommissionTests(unittest.TestCase):
+    def test_official_compose_keeps_docker_delete_disabled(self):
+        compose = (AGENT_ROOT / "docker-compose.yml").read_text(encoding="utf-8")
+        self.assertIn("DELETE=0", compose)
+        self.assertNotIn("DELETE=1", compose)
+
+    def test_tombstone_is_atomic_private_and_secret_free(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = os.path.join(directory, "decommission.json")
+            value = decommission.persist_tombstone("operation", "agent", "prepared", path)
+            self.assertEqual(value["phase"], "prepared")
+            self.assertEqual(decommission.load_tombstone(path)["operation_id"], "operation")
+            self.assertEqual(stat.S_IMODE(os.stat(path).st_mode), 0o600)
+            self.assertNotIn("token", str(value).lower())
+
+    def test_tombstone_cannot_change_operation_or_downgrade_finalized_state(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = os.path.join(directory, "decommission.json")
+            decommission.persist_tombstone("operation", "agent", "prepared", path)
+            decommission.persist_tombstone("operation", "agent", "finalized", path)
+            with self.assertRaisesRegex(ValueError, "decommission_tombstone_conflict"):
+                decommission.persist_tombstone("other-operation", "agent", "prepared", path)
+            with self.assertRaisesRegex(ValueError, "decommission_tombstone_conflict"):
+                decommission.persist_tombstone("operation", "agent", "prepared", path)
+
+    def test_tunnel_is_stopped_but_never_removed(self):
+        tunnel = FakeContainer("a" * 64, "dockflare-agent-tunnel", image="cloudflare/cloudflared:latest")
+        result, image = decommission.stop_tunnel_container(FakeDocker([tunnel]))
+        self.assertEqual(result, "stopped")
+        self.assertEqual(image, "cloudflare/cloudflared:latest")
+        self.assertEqual(tunnel.stop_calls, 1)
+        self.assertFalse(hasattr(tunnel, "remove_calls"))
+
+    def test_already_stopped_and_absent_are_idempotent(self):
+        stopped = FakeContainer("b" * 64, "dockflare-agent-tunnel", status="exited")
+        result, _image = decommission.stop_tunnel_container(FakeDocker([stopped]))
+        self.assertEqual(result, "stopped")
+        self.assertEqual(stopped.stop_calls, 0)
+        result, _image = decommission.stop_tunnel_container(FakeDocker([]))
+        self.assertEqual(result, "absent")
+
+    def test_duplicate_tunnel_stop_has_no_second_stop_side_effect(self):
+        tunnel = FakeContainer("f" * 64, "dockflare-agent-tunnel")
+        client = FakeDocker([tunnel])
+        self.assertEqual(decommission.stop_tunnel_container(client)[0], "stopped")
+        self.assertEqual(decommission.stop_tunnel_container(client)[0], "stopped")
+        self.assertEqual(tunnel.stop_calls, 1)
+
+    def test_self_target_requires_runtime_id_and_expected_identity(self):
+        runtime_id = "c" * 64
+        expected = FakeContainer(runtime_id, "dockflare-agent", service="dockflare-agent", image="alplat/dockflare-agent:dev")
+        unrelated = FakeContainer("d" * 64, "database", service="database")
+        client = FakeDocker([expected, unrelated])
+        self.assertIs(decommission.resolve_self_container(client, runtime_id[:12]), expected)
+        self.assertIsNone(decommission.resolve_self_container(client, unrelated.id[:12]))
+        self.assertIsNone(decommission.resolve_self_container(client, "dockflare-agent"))
+
+    def test_self_stop_is_scheduled_without_delete(self):
+        runtime_id = "e" * 64
+        expected = FakeContainer(runtime_id, "dockflare-agent", service="dockflare-agent")
+        with patch.object(decommission, "Thread") as thread:
+            self.assertTrue(decommission.schedule_self_stop(FakeDocker([expected]), runtime_id[:12]))
+        thread.assert_called_once()
+        self.assertTrue(thread.call_args.kwargs["daemon"])
+
+
+if __name__ == "__main__":
+    unittest.main()

dockflare/.gitignore

@@ -86,4 +86,5 @@ data/state.json
 *.bak
 *.swp
 *.log
-logs/
+logs/
+tests

dockflare/app/__init__.py

@@ -20,6 +20,7 @@ import queue
 import sys
 import os
 import json
+import hashlib
 
 from flask import Flask
 from flask_wtf.csrf import CSRFProtect
@@ -135,8 +136,19 @@ except Exception as e:
     logging.error(f"FATAL: Failed to connect to Docker daemon: {e}")
     docker_client = None 
 
+
+def get_static_asset_version(path):
+    try:
+        with open(path, "rb") as asset_file:
+            return hashlib.sha256(asset_file.read()).hexdigest()[:12]
+    except OSError:
+        return config.APP_VERSION.lstrip("v")
+
+
 def create_app():
     app_instance = Flask(__name__)
+    main_js_path = os.path.join(app_instance.static_folder, "js", "main.js")
+    main_js_version = get_static_asset_version(main_js_path)
     app_instance.secret_key = os.urandom(24)
     app_instance.config['PREFERRED_URL_SCHEME'] = 'http'
     app_instance.config['APP_VERSION'] = config.APP_VERSION
@@ -220,7 +232,10 @@ def create_app():
     @app_instance.context_processor
     def inject_version():
         """Injects the app version into all templates."""
-        return dict(app_version=config.APP_VERSION)
+        return {
+            "app_version": config.APP_VERSION,
+            "main_js_version": main_js_version,
+        }
 
     app_instance.reconciliation_info = {
         "in_progress": False,

dockflare/app/config.py

@@ -86,6 +86,15 @@ CUSTOM_LABEL_PREFIX = os.getenv('LABEL_PREFIX')
 LABEL_PREFIX = CUSTOM_LABEL_PREFIX or PRIMARY_LABEL_PREFIX
 
 CLEANUP_INTERVAL_SECONDS = int(os.getenv('CLEANUP_INTERVAL_SECONDS', 60))
+TUNNEL_SYNC_RETRY_SECONDS = _get_int_env('TUNNEL_SYNC_RETRY_SECONDS', 60, minimum=1)
+AGENT_MAX_REQUEST_BYTES = _get_int_env('AGENT_MAX_REQUEST_BYTES', 4 * 1024 * 1024, minimum=1024)
+AGENT_MAX_CONTAINERS = _get_int_env('AGENT_MAX_CONTAINERS', 5000, minimum=1)
+AGENT_MAX_LABELS = _get_int_env('AGENT_MAX_LABELS', 256, minimum=1)
+AGENT_MAX_LABEL_KEY_BYTES = _get_int_env('AGENT_MAX_LABEL_KEY_BYTES', 256, minimum=1)
+AGENT_MAX_LABEL_VALUE_BYTES = _get_int_env('AGENT_MAX_LABEL_VALUE_BYTES', 16 * 1024, minimum=1)
+AGENT_MAX_CONTAINER_ID_BYTES = _get_int_env('AGENT_MAX_CONTAINER_ID_BYTES', 128, minimum=1)
+AGENT_MAX_CONTAINER_NAME_BYTES = _get_int_env('AGENT_MAX_CONTAINER_NAME_BYTES', 255, minimum=1)
+AGENT_MAX_EVENT_TYPE_BYTES = _get_int_env('AGENT_MAX_EVENT_TYPE_BYTES', 64, minimum=1)
 AGENT_STATUS_UPDATE_INTERVAL_SECONDS = int(os.getenv('AGENT_STATUS_UPDATE_INTERVAL_SECONDS', 30))
 STATE_FILE_PATH = os.getenv('STATE_FILE_PATH', '/app/data/state.json')
 MAX_LOG_QUEUE_SIZE = 200
@@ -148,9 +157,13 @@ DOCKFLARE_PUBLIC_URL = os.getenv('DOCKFLARE_PUBLIC_URL', '')
 
 EMAIL_ENABLED = False
 EMAIL_CONFIG = {}
+NOTIFICATION_CONFIG = {}
+NOTIFICATION_QUEUE_SIZE = _get_int_env('NOTIFICATION_QUEUE_SIZE', 100, minimum=1)
+NOTIFICATION_MAX_BODY_CHARS = _get_int_env('NOTIFICATION_MAX_BODY_CHARS', 4000, minimum=256)
+NOTIFICATION_RESOURCE_LIST_LIMIT = _get_int_env('NOTIFICATION_RESOURCE_LIST_LIMIT', 10, minimum=1)
+NOTIFICATION_HEALTH_INTERVAL_SECONDS = _get_int_env('NOTIFICATION_HEALTH_INTERVAL_SECONDS', 15, minimum=5)
 MAIL_MANAGER_INTERNAL_URL = os.getenv('MAIL_MANAGER_INTERNAL_URL', 'http://dockflare-mail-manager:8025')
 EMAIL_JWT_ALGORITHM = 'EdDSA'
 EMAIL_JWT_ISSUER = 'dockflare-master'
 EMAIL_JWT_AUDIENCE = 'dockflare-mail'
 EMAIL_JWT_EXPIRY_SECONDS = 3600
-

dockflare/app/core/agent_decommission.py

@@ -0,0 +1,841 @@
+"""Durable, ownership-safe DockFlare Agent decommission orchestration."""
+
+import copy
+import logging
+import os
+import re
+import uuid
+from datetime import datetime, timedelta, timezone
+
+from app import config, tunnel_state
+from app.core.access_manager import delete_cloudflare_access_application
+from app.core.cloudflare_api import delete_cloudflare_dns_record, delete_tunnel_via_api
+from app.core.state_manager import (
+    agent_decommissions,
+    add_agent_key,
+    agents,
+    list_agent_keys,
+    managed_rules,
+    revoke_agent_key,
+    save_state,
+    state_lock,
+)
+from app.core.tunnel_manager import update_cloudflare_config
+from app.core import notification_manager
+
+
+TERMINAL_STATES = {"completed", "forced_completed"}
+RETRYABLE_STATES = {"prepare_failed", "cleanup_failed", "finalize_failed", "force_failed", "timed_out"}
+FORCEABLE_STATES = {
+    "waiting_for_prepare", "prepare_failed", "timed_out", "cleanup_failed",
+    "waiting_for_finalize", "finalize_failed",
+}
+IMAGE_REFERENCE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@-]{0,254}$")
+
+
+def _bounded_int_env(name, default, minimum, maximum):
+    try:
+        return max(minimum, min(maximum, int(os.getenv(name, str(default)))))
+    except (TypeError, ValueError):
+        return default
+
+
+DEFAULT_TIMEOUT_SECONDS = _bounded_int_env("AGENT_DECOMMISSION_TIMEOUT_SECONDS", 180, 30, 1800)
+
+
+def notification_context(operation):
+    plan = operation.get("resource_plan") or {}
+    results = operation.get("cleanup_results") or {}
+    return {
+        "agent_name": operation.get("display_name"),
+        "agent_id": str(operation.get("agent_id") or "")[:12],
+        "tunnel_id": str(plan.get("tunnel_id") or "")[:12],
+        "operation_id": str(operation.get("operation_id") or "")[:12],
+        "operation": "decommission",
+        "rules_count": results.get("rules_removed", len(plan.get("rule_keys") or [])),
+        "source": "admin",
+        "public_url": config.DOCKFLARE_PUBLIC_URL,
+    }
+
+
+def notify_transition(operation, previous_state=None):
+    """Emit a safe notification for a committed decommission state transition."""
+    if not isinstance(operation, dict):
+        return False
+    state = operation.get("state")
+    if state == previous_state:
+        return False
+    if state in TERMINAL_STATES:
+        event_type = "agent.decommission_completed"
+    elif state == "timed_out":
+        event_type = "agent.decommission_stalled"
+    elif state in {"prepare_failed", "cleanup_failed", "finalize_failed", "force_failed"}:
+        event_type = "agent.decommission_failed"
+    else:
+        return False
+    return notification_manager.emit(
+        event_type,
+        str(operation.get("operation_id") or operation.get("agent_id") or "unknown"),
+        notification_context(operation),
+    )
+
+
+class DecommissionError(RuntimeError):
+    """Stable decommission failure safe to map to an API response."""
+
+    def __init__(self, code, http_status=409):
+        super().__init__(code)
+        self.code = code
+        self.http_status = http_status
+
+
+def _utcnow():
+    return datetime.now(timezone.utc)
+
+
+def _iso(value=None):
+    return (value or _utcnow()).isoformat()
+
+
+def _deadline():
+    return _iso(_utcnow() + timedelta(seconds=DEFAULT_TIMEOUT_SECONDS))
+
+
+def _valid_image_reference(value):
+    if not isinstance(value, str):
+        return None
+    candidate = value.strip()
+    return candidate if IMAGE_REFERENCE_RE.fullmatch(candidate) else None
+
+
+def _agent_rules(agent_id):
+    return {
+        key: copy.deepcopy(rule)
+        for key, rule in managed_rules.items()
+        if rule.get("source") == "agent" and rule.get("agent_id") == agent_id
+    }
+
+
+def _tunnel_disposition(agent_id, agent, rules):
+    tunnel_id = agent.get("assigned_tunnel_id")
+    ownership = agent.get("assigned_tunnel_ownership", "unknown")
+    if not tunnel_id or ownership != "created_exclusive":
+        return "preserve_adopted" if ownership == "adopted" else "unknown"
+    if tunnel_state.get("id") == tunnel_id:
+        return "preserve_shared"
+    if any(
+        other_id != agent_id and other.get("assigned_tunnel_id") == tunnel_id
+        for other_id, other in agents.items()
+    ):
+        return "preserve_shared"
+    owned_keys = set(rules)
+    if any(
+        key not in owned_keys and rule.get("tunnel_id") == tunnel_id
+        for key, rule in managed_rules.items()
+    ):
+        return "preserve_shared"
+    return "delete_exclusive"
+
+
+def _build_resource_plan(agent_id, agent):
+    rules = _agent_rules(agent_id)
+    tunnel_id = agent.get("assigned_tunnel_id")
+    dns_targets = sorted({
+        (rule.get("zone_id"), rule.get("hostname"))
+        for rule in rules.values()
+        if rule.get("zone_id") and rule.get("hostname") and not rule.get("hostname", "").startswith("*.")
+    })
+    return {
+        "tunnel_id": tunnel_id,
+        "tunnel_disposition": _tunnel_disposition(agent_id, agent, rules),
+        "rule_keys": sorted(rules),
+        "dns_targets": [{"zone_id": zone_id, "hostname": hostname} for zone_id, hostname in dns_targets],
+        "access_app_ids": sorted({
+            rule.get("access_app_id") for rule in rules.values() if rule.get("access_app_id")
+        }),
+    }
+
+
+def preview_decommission(agent_id):
+    """Return a non-mutating, secret-safe preflight view for the admin UI."""
+    with state_lock:
+        agent = agents.get(agent_id)
+        if not agent:
+            raise DecommissionError("agent_not_found", 404)
+        plan = _build_resource_plan(agent_id, agent)
+        return {
+            "agent_id": agent_id,
+            "display_name": str(agent.get("display_name") or f"agent-{agent_id[:8]}")[:128],
+            "last_seen": agent.get("last_seen"),
+            "assigned_tunnel_name": agent.get("assigned_tunnel_name"),
+            "resource_plan": {
+                "tunnel_disposition": plan["tunnel_disposition"],
+                "rule_count": len(plan["rule_keys"]),
+                "dns_record_count": len(plan["dns_targets"]),
+                "access_app_count": len(plan["access_app_ids"]),
+            },
+            "remote_actions": {
+                "tunnel_container": "stop_only",
+                "agent_container": (
+                    "stop_scheduled"
+                    if "self_stop.v1" in (agent.get("capabilities") or [])
+                    else "manual"
+                ),
+            },
+            "container_names": ["dockflare-agent-tunnel", "dockflare-agent"],
+            "deployment_directory": "$HOME/dockflare-agent",
+            "preserve_networks": ["cloudflare-net"],
+        }
+
+
+def _new_operation(agent_id, agent):
+    operation_id = str(uuid.uuid4())
+    command_id = str(uuid.uuid4())
+    now = _iso()
+    return {
+        "operation_id": operation_id,
+        "agent_id": agent_id,
+        "display_name": str(agent.get("display_name") or f"agent-{agent_id[:8]}")[:128],
+        "state": "waiting_for_prepare",
+        "requested_at": now,
+        "updated_at": now,
+        "deadline_at": _deadline(),
+        "prepare_command_id": command_id,
+        "finalize_command_id": None,
+        "requested_self_action": "stop",
+        "agent_capabilities": copy.deepcopy(agent.get("capabilities") or []),
+        "remote_results": {
+            "tombstone_persisted": False,
+            "tunnel_container": "pending",
+            "agent_container": "stop_pending",
+            "host_container_removal_required": True,
+        },
+        "resource_plan": _build_resource_plan(agent_id, agent),
+        "cleanup_results": {},
+        "host_cleanup_plan": {
+            "agent_container_name": "dockflare-agent",
+            "tunnel_container_name": "dockflare-agent-tunnel",
+            "socket_proxy_container_name": "dockflare-socket-proxy",
+            "compose_services": ["docker-socket-proxy", "dockflare-init", "dockflare-agent"],
+            "compose_filename": "docker-compose.yml",
+            "deployment_directory": "$HOME/dockflare-agent",
+            "agent_image": None,
+            "cloudflared_image": None,
+            "preserve_networks": ["cloudflare-net"],
+        },
+        "last_error_code": None,
+        "retry_count": 0,
+        "forced": False,
+        "durable_command": {
+            "action": "prepare_decommission",
+            "protocol_version": 1,
+            "operation_id": operation_id,
+            "command_id": command_id,
+            "requested_self_action": "stop",
+            "expected_tunnel_id": agent.get("assigned_tunnel_id"),
+            "created_at": now,
+        },
+        "acknowledged_commands": [],
+    }
+
+
+def start_decommission(agent_id):
+    """Persist or return one active decommission operation for an Agent."""
+    with state_lock:
+        agent = agents.get(agent_id)
+        if not agent:
+            raise DecommissionError("agent_not_found", 404)
+        for operation in agent_decommissions.values():
+            if operation.get("agent_id") == agent_id and operation.get("state") not in TERMINAL_STATES:
+                return copy.deepcopy(operation), False
+        operation = _new_operation(agent_id, agent)
+        previous_agent = copy.deepcopy(agent)
+        agent_decommissions[operation["operation_id"]] = operation
+        agent.update({
+            "decommission_operation_id": operation["operation_id"],
+            "decommission_state": operation["state"],
+        })
+        if not save_state():
+            agent.clear()
+            agent.update(previous_agent)
+            agent_decommissions.pop(operation["operation_id"], None)
+            raise DecommissionError("persistence_failed", 503)
+        return copy.deepcopy(operation), True
+
+
+def get_operation(operation_id):
+    with state_lock:
+        operation = agent_decommissions.get(operation_id)
+        return copy.deepcopy(operation) if operation else None
+
+
+def command_for_agent(agent_id):
+    with state_lock:
+        agent = agents.get(agent_id)
+        if not agent:
+            return None
+        operation_id = agent.get("decommission_operation_id")
+        operation = agent_decommissions.get(operation_id)
+        if not operation or operation.get("state") in TERMINAL_STATES:
+            return None
+        command = operation.get("durable_command")
+        return copy.deepcopy(command) if command else None
+
+
+def _set_failure(operation_id, state, code):
+    with state_lock:
+        operation = agent_decommissions.get(operation_id)
+        if not operation:
+            return
+        previous_operation = copy.deepcopy(operation)
+        operation["state"] = state
+        operation["last_error_code"] = code
+        operation["updated_at"] = _iso()
+        agent = agents.get(operation.get("agent_id"))
+        previous_agent = copy.deepcopy(agent) if agent else None
+        if agent:
+            agent["decommission_state"] = state
+        if not save_state():
+            operation.clear()
+            operation.update(previous_operation)
+            if agent is not None and previous_agent is not None:
+                agent.clear()
+                agent.update(previous_agent)
+            logging.error("AGENT_DECOMMISSION: failed to persist %s for %s", state, operation_id)
+
+
+def _transition_to_finalize(operation_id, cleanup_results):
+    with state_lock:
+        operation = agent_decommissions.get(operation_id)
+        if not operation:
+            raise DecommissionError("operation_not_found", 404)
+        previous_operation = copy.deepcopy(operation)
+        agent = agents.get(operation["agent_id"])
+        previous_agent = copy.deepcopy(agent) if agent else None
+        final_command_id = operation.get("finalize_command_id") or str(uuid.uuid4())
+        command = {
+            "action": "finalize_decommission",
+            "protocol_version": 1,
+            "operation_id": operation_id,
+            "command_id": final_command_id,
+            "requested_self_action": "stop",
+        }
+        operation.update({
+            "state": "waiting_for_finalize",
+            "updated_at": _iso(),
+            "deadline_at": _deadline(),
+            "finalize_command_id": final_command_id,
+            "durable_command": command,
+            "cleanup_results": cleanup_results,
+            "last_error_code": None,
+        })
+        if agent:
+            agent["decommission_state"] = "waiting_for_finalize"
+        if not save_state():
+            operation.clear()
+            operation.update(previous_operation)
+            if agent is not None and previous_agent is not None:
+                agent.clear()
+                agent.update(previous_agent)
+            raise DecommissionError("persistence_failed", 503)
+
+
+def _cleanup_snapshot(operation_id):
+    with state_lock:
+        operation = agent_decommissions.get(operation_id)
+        if not operation:
+            raise DecommissionError("operation_not_found", 404)
+        if operation.get("state") not in {"remote_prepared", "cleanup_failed", "force_cleanup"}:
+            raise DecommissionError("operation_busy")
+        agent_id = operation["agent_id"]
+        agent = agents.get(agent_id)
+        if not agent:
+            raise DecommissionError("agent_not_found", 404)
+        previous_operation = copy.deepcopy(operation)
+        previous_agent = copy.deepcopy(agent)
+        rules = _agent_rules(agent_id)
+        plan = _build_resource_plan(agent_id, agent)
+        operation["state"] = "master_cleanup"
+        operation["resource_plan"] = plan
+        operation["updated_at"] = _iso()
+        agent["decommission_state"] = "master_cleanup"
+        if not save_state():
+            operation.clear()
+            operation.update(previous_operation)
+            agent.clear()
+            agent.update(previous_agent)
+            raise DecommissionError("persistence_failed", 503)
+        return agent_id, rules, plan
+
+
+def _assert_cleanup_current(operation_id, agent_id):
+    with state_lock:
+        operation = agent_decommissions.get(operation_id)
+        agent = agents.get(agent_id)
+        if (
+            not operation
+            or operation.get("state") != "master_cleanup"
+            or not agent
+            or agent.get("decommission_operation_id") != operation_id
+        ):
+            raise DecommissionError("operation_changed")
+
+
+def _cleanup_access_apps(operation_id, agent_id, owned_rules, app_ids):
+    deleted = 0
+    for app_id in app_ids:
+        _assert_cleanup_current(operation_id, agent_id)
+        with state_lock:
+            shared = any(
+                key not in owned_rules and rule.get("access_app_id") == app_id
+                for key, rule in managed_rules.items()
+            )
+        if shared:
+            continue
+        if not delete_cloudflare_access_application(app_id):
+            raise DecommissionError("access_cleanup_failed")
+        deleted += 1
+    return deleted
+
+
+def _cleanup_dns_records(operation_id, agent_id, tunnel_id, targets):
+    deleted = 0
+    for target in targets:
+        _assert_cleanup_current(operation_id, agent_id)
+        with state_lock:
+            still_owned = any(
+                rule.get("source") == "agent"
+                and rule.get("agent_id") == agent_id
+                and rule.get("zone_id") == target["zone_id"]
+                and rule.get("hostname") == target["hostname"]
+                and rule.get("tunnel_id") == tunnel_id
+                for rule in managed_rules.values()
+            )
+            shared_hostname = any(
+                rule.get("agent_id") != agent_id
+                and rule.get("zone_id") == target["zone_id"]
+                and rule.get("hostname") == target["hostname"]
+                for rule in managed_rules.values()
+            )
+        if not still_owned or shared_hostname:
+            continue
+        if not delete_cloudflare_dns_record(target["zone_id"], target["hostname"], tunnel_id):
+            raise DecommissionError("dns_cleanup_failed")
+        deleted += 1
+    return deleted
+
+
+def _cleanup_tunnel(operation_id, agent_id, tunnel_id):
+    _assert_cleanup_current(operation_id, agent_id)
+    with state_lock:
+        agent = agents.get(agent_id)
+        disposition = _tunnel_disposition(agent_id, agent, _agent_rules(agent_id)) if agent else "unknown"
+    if not tunnel_id or disposition != "delete_exclusive":
+        return disposition
+    if not delete_tunnel_via_api(tunnel_id):
+        raise DecommissionError("tunnel_cleanup_failed")
+    return "deleted"
+
+
+def _remove_agent_rules(operation_id, agent_id):
+    _assert_cleanup_current(operation_id, agent_id)
+    with state_lock:
+        current_rules = _agent_rules(agent_id)
+        previous_rules = {key: managed_rules.get(key) for key in current_rules}
+        for key in current_rules:
+            managed_rules.pop(key, None)
+        if not save_state():
+            managed_rules.update({key: rule for key, rule in previous_rules.items() if rule is not None})
+            raise DecommissionError("persistence_failed", 503)
+    return len(current_rules), previous_rules
+
+
+def _restore_rules(previous_rules):
+    with state_lock:
+        managed_rules.update({key: rule for key, rule in previous_rules.items() if rule is not None})
+        save_state()
+
+
+def run_master_cleanup(operation_id):
+    """Delete only resources still proven to belong exclusively to this Agent."""
+    agent_id, rules, plan = _cleanup_snapshot(operation_id)
+    results = {"access_apps_deleted": 0, "dns_records_deleted": 0, "rules_removed": 0, "tunnel": "preserved"}
+    try:
+        tunnel_id = plan.get("tunnel_id")
+        results["access_apps_deleted"] = _cleanup_access_apps(
+            operation_id, agent_id, rules, plan["access_app_ids"]
+        )
+        results["dns_records_deleted"] = _cleanup_dns_records(
+            operation_id, agent_id, tunnel_id, plan["dns_targets"]
+        )
+        results["tunnel"] = _cleanup_tunnel(operation_id, agent_id, tunnel_id)
+        results["rules_removed"], previous_rules = _remove_agent_rules(operation_id, agent_id)
+        if tunnel_id and results["tunnel"] != "deleted" and not update_cloudflare_config(tunnel_id):
+            _restore_rules(previous_rules)
+            raise DecommissionError("tunnel_sync_failed")
+        _transition_to_finalize(operation_id, results)
+        return get_operation(operation_id)
+    except DecommissionError as error:
+        _set_failure(operation_id, "cleanup_failed", error.code)
+        raise
+    except Exception:
+        logging.exception("AGENT_DECOMMISSION: cleanup failed for %s", operation_id)
+        _set_failure(operation_id, "cleanup_failed", "cleanup_failed")
+        raise DecommissionError("cleanup_failed")
+
+
+def _validate_ack(operation, agent_id, payload):
+    if operation.get("agent_id") != agent_id:
+        raise DecommissionError("operation_agent_mismatch", 403)
+    command = operation.get("durable_command") or {}
+    if payload.get("command_id") != command.get("command_id"):
+        raise DecommissionError("stale_command")
+    if payload.get("operation_id") != operation.get("operation_id"):
+        raise DecommissionError("stale_operation")
+
+
+def record_ack(agent_id, operation_id, payload):
+    """Persist an Agent acknowledgement, returning whether cleanup should run."""
+    with state_lock:
+        operation = agent_decommissions.get(operation_id)
+        if not operation:
+            raise DecommissionError("operation_not_found", 404)
+        command_id = payload.get("command_id")
+        if not command_id:
+            raise DecommissionError("invalid_payload", 400)
+        if command_id in operation.get("acknowledged_commands", []):
+            return copy.deepcopy(operation), False
+        _validate_ack(operation, agent_id, payload)
+
+        previous_operation = copy.deepcopy(operation)
+        agent = agents.get(agent_id)
+        previous_agent = copy.deepcopy(agent) if agent else None
+        phase = payload.get("phase")
+        if phase == "prepared":
+            if operation.get("state") not in {"waiting_for_prepare", "prepare_failed", "timed_out"}:
+                raise DecommissionError("invalid_operation_state")
+            tunnel_result = payload.get("tunnel_container")
+            if payload.get("tombstone_persisted") is not True or tunnel_result not in {"stopped", "absent"}:
+                operation["state"] = "prepare_failed"
+                operation["last_error_code"] = str(payload.get("error_code") or "prepare_failed")[:64]
+                operation["updated_at"] = _iso()
+                if not save_state():
+                    operation.clear()
+                    operation.update(previous_operation)
+                    if agent is not None and previous_agent is not None:
+                        agent.clear()
+                        agent.update(previous_agent)
+                    raise DecommissionError("persistence_failed", 503)
+                raise DecommissionError(operation["last_error_code"])
+            cloudflared_image = _valid_image_reference(payload.get("cloudflared_image"))
+            agent_image = _valid_image_reference(payload.get("agent_image"))
+            self_stop_capability = payload.get("self_stop_capability")
+            if not cloudflared_image or (self_stop_capability == "supported" and not agent_image):
+                raise DecommissionError("invalid_image_reference", 400)
+            operation["remote_results"].update({
+                "tombstone_persisted": True,
+                "tunnel_container": tunnel_result,
+                "self_stop_capability": self_stop_capability if self_stop_capability in {"supported", "manual"} else "manual",
+            })
+            operation["host_cleanup_plan"]["agent_image"] = agent_image
+            operation["host_cleanup_plan"]["cloudflared_image"] = cloudflared_image
+            operation["state"] = "remote_prepared"
+            should_cleanup = True
+        elif phase == "shutdown_scheduled":
+            if operation.get("state") not in {"waiting_for_finalize", "finalize_failed", "timed_out"}:
+                raise DecommissionError("invalid_operation_state")
+            self_stop_capability = payload.get("self_stop_capability")
+            operation["remote_results"]["self_stop_capability"] = (
+                "supported" if self_stop_capability == "supported" else "manual"
+            )
+            operation["remote_results"]["agent_container"] = (
+                "stop_scheduled" if self_stop_capability == "supported" else "manual_required"
+            )
+            operation["state"] = "shutdown_scheduled"
+            should_cleanup = False
+        else:
+            raise DecommissionError("invalid_ack_phase", 400)
+
+        operation.setdefault("acknowledged_commands", []).append(command_id)
+        operation["durable_command"] = None
+        operation["last_error_code"] = None
+        operation["updated_at"] = _iso()
+        if agent:
+            agent["decommission_state"] = operation["state"]
+        if not save_state():
+            operation.clear()
+            operation.update(previous_operation)
+            if agent is not None and previous_agent is not None:
+                agent.clear()
+                agent.update(previous_agent)
+            raise DecommissionError("persistence_failed", 503)
+        return copy.deepcopy(operation), should_cleanup
+
+
+def _revoke_bound_key(agent_id):
+    receipts = []
+    for token, metadata in list_agent_keys().items():
+        if metadata.get("bound_agent_id") != agent_id or metadata.get("status") == "revoked":
+            continue
+        if not revoke_agent_key(token):
+            _restore_bound_key(receipts)
+            return False, []
+        receipts.append((token, copy.deepcopy(metadata)))
+    return True, receipts
+
+
+def _restore_bound_key(revocation_receipt):
+    for token, metadata in revocation_receipt or []:
+        add_agent_key(token, metadata)
+
+
+def complete_finalization(operation_id):
+    with state_lock:
+        operation = agent_decommissions.get(operation_id)
+        if not operation or operation.get("state") != "shutdown_scheduled":
+            raise DecommissionError("invalid_operation_state")
+        agent_id = operation["agent_id"]
+    key_revoked, revocation_receipt = _revoke_bound_key(agent_id)
+    if not key_revoked:
+        _set_failure(operation_id, "finalize_failed", "key_revocation_failed")
+        raise DecommissionError("key_revocation_failed")
+    with state_lock:
+        operation = agent_decommissions[operation_id]
+        previous_agent = agents.pop(agent_id, None)
+        operation.update({"state": "completed", "updated_at": _iso(), "last_error_code": None})
+        if not save_state():
+            if previous_agent is not None:
+                agents[agent_id] = previous_agent
+            _restore_bound_key(revocation_receipt)
+            operation["state"] = "finalize_failed"
+            operation["last_error_code"] = "persistence_failed"
+            raise DecommissionError("persistence_failed", 503)
+        return copy.deepcopy(operation)
+
+
+def retry_operation(operation_id):
+    with state_lock:
+        operation = agent_decommissions.get(operation_id)
+        if not operation:
+            raise DecommissionError("operation_not_found", 404)
+        if operation.get("state") not in RETRYABLE_STATES:
+            raise DecommissionError("operation_not_retryable")
+        previous_operation = copy.deepcopy(operation)
+        agent = agents.get(operation["agent_id"])
+        previous_agent = copy.deepcopy(agent) if agent else None
+        operation["retry_count"] = int(operation.get("retry_count") or 0) + 1
+        operation["last_error_code"] = None
+        operation["updated_at"] = _iso()
+        retry_force = False
+        if operation["state"] == "force_failed":
+            if not save_state():
+                operation.clear()
+                operation.update(previous_operation)
+                raise DecommissionError("persistence_failed", 503)
+            retry_force = True
+            rerun_cleanup = False
+        elif operation["state"] == "cleanup_failed":
+            if not save_state():
+                operation.clear()
+                operation.update(previous_operation)
+                raise DecommissionError("persistence_failed", 503)
+            rerun_cleanup = True
+        else:
+            command = "prepare_decommission" if not operation.get("finalize_command_id") else "finalize_decommission"
+            command_id = str(uuid.uuid4())
+            operation["durable_command"] = {
+                "action": command,
+                "protocol_version": 1,
+                "operation_id": operation_id,
+                "command_id": command_id,
+                "requested_self_action": "stop",
+            }
+            if command == "prepare_decommission":
+                operation["prepare_command_id"] = command_id
+                operation["state"] = "waiting_for_prepare"
+            else:
+                operation["finalize_command_id"] = command_id
+                operation["state"] = "waiting_for_finalize"
+            operation["deadline_at"] = _deadline()
+            if agent:
+                agent["decommission_state"] = operation["state"]
+            if not save_state():
+                operation.clear()
+                operation.update(previous_operation)
+                if agent is not None and previous_agent is not None:
+                    agent.clear()
+                    agent.update(previous_agent)
+                raise DecommissionError("persistence_failed", 503)
+            rerun_cleanup = False
+            retry_force = False
+    if retry_force:
+        return force_cleanup(operation_id)
+    return run_master_cleanup(operation_id) if rerun_cleanup else get_operation(operation_id)
+
+
+def force_cleanup(operation_id):
+    with state_lock:
+        operation = agent_decommissions.get(operation_id)
+        if not operation:
+            raise DecommissionError("operation_not_found", 404)
+        if operation.get("state") in TERMINAL_STATES:
+            return copy.deepcopy(operation)
+        if operation.get("state") not in FORCEABLE_STATES and operation.get("state") != "force_failed":
+            raise DecommissionError("operation_not_forceable")
+        previous_operation = copy.deepcopy(operation)
+        operation["forced"] = True
+        operation["state"] = "force_cleanup"
+        operation["updated_at"] = _iso()
+        if not save_state():
+            operation.clear()
+            operation.update(previous_operation)
+            raise DecommissionError("persistence_failed", 503)
+        agent_id = operation["agent_id"]
+    key_revoked, _ = _revoke_bound_key(agent_id)
+    if not key_revoked:
+        _set_failure(operation_id, "force_failed", "key_revocation_failed")
+        raise DecommissionError("key_revocation_failed")
+    try:
+        run_master_cleanup(operation_id)
+    except DecommissionError:
+        _set_failure(operation_id, "force_failed", "force_cleanup_failed")
+        raise
+    with state_lock:
+        operation = agent_decommissions[operation_id]
+        previous_operation = copy.deepcopy(operation)
+        previous_agent = agents.get(agent_id)
+        operation.update({
+            "state": "forced_completed",
+            "updated_at": _iso(),
+            "durable_command": None,
+            "last_error_code": None,
+        })
+        operation["remote_results"]["remote_host_cleanup_required"] = True
+        agents.pop(agent_id, None)
+        if not save_state():
+            operation.clear()
+            operation.update(previous_operation)
+            if previous_agent is not None:
+                agents[agent_id] = previous_agent
+                previous_agent["decommission_state"] = "force_failed"
+            operation["state"] = "force_failed"
+            operation["last_error_code"] = "persistence_failed"
+            raise DecommissionError("persistence_failed", 503)
+        return copy.deepcopy(operation)
+
+
+def expire_due_operations(now=None):
+    """Persist bounded prepare/finalize timeouts without triggering force cleanup."""
+    current = now or _utcnow()
+    changed = []
+    with state_lock:
+        previous_operations = {}
+        previous_agents = {}
+        for operation_id, operation in agent_decommissions.items():
+            if operation.get("state") not in {"waiting_for_prepare", "waiting_for_finalize"}:
+                continue
+            try:
+                deadline = datetime.fromisoformat(str(operation.get("deadline_at")).replace("Z", "+00:00"))
+                if deadline.tzinfo is None:
+                    deadline = deadline.replace(tzinfo=timezone.utc)
+            except (TypeError, ValueError):
+                continue
+            if deadline > current:
+                continue
+            previous_operations[operation_id] = copy.deepcopy(operation)
+            agent = agents.get(operation.get("agent_id"))
+            if agent:
+                previous_agents[operation["agent_id"]] = copy.deepcopy(agent)
+                agent["decommission_state"] = "timed_out"
+            operation["state"] = "timed_out"
+            operation["last_error_code"] = "agent_timeout"
+            operation["updated_at"] = _iso(current)
+            changed.append(operation_id)
+        if changed and not save_state():
+            for operation_id, previous in previous_operations.items():
+                agent_decommissions[operation_id].clear()
+                agent_decommissions[operation_id].update(previous)
+            for agent_id, previous in previous_agents.items():
+                agents[agent_id].clear()
+                agents[agent_id].update(previous)
+            return []
+    for operation_id in changed:
+        notify_transition(get_operation(operation_id), "waiting")
+    return changed
+
+
+def prune_completed_operations(now=None, retention_days=30):
+    current = now or _utcnow()
+    removed = {}
+    with state_lock:
+        for operation_id, operation in list(agent_decommissions.items()):
+            if operation.get("state") not in TERMINAL_STATES:
+                continue
+            try:
+                updated = datetime.fromisoformat(str(operation.get("updated_at")).replace("Z", "+00:00"))
+                if updated.tzinfo is None:
+                    updated = updated.replace(tzinfo=timezone.utc)
+            except (TypeError, ValueError):
+                continue
+            if current - updated < timedelta(days=retention_days):
+                continue
+            removed[operation_id] = agent_decommissions.pop(operation_id)
+        if removed and not save_state():
+            agent_decommissions.update(removed)
+            return []
+    return sorted(removed)
+
+
+def timeout_worker(stop_event, interval_seconds=5):
+    while not stop_event.is_set():
+        try:
+            expire_due_operations()
+            prune_completed_operations()
+        except Exception:
+            logging.exception("AGENT_DECOMMISSION: timeout worker failed")
+        stop_event.wait(interval_seconds)
+
+
+def serialize_operation(operation):
+    """Return a public, secret-safe operation representation."""
+    if not operation:
+        return None
+    allowed = {
+        "operation_id", "agent_id", "display_name", "state", "requested_at", "updated_at",
+        "deadline_at", "requested_self_action", "remote_results", "resource_plan",
+        "cleanup_results", "host_cleanup_plan", "last_error_code", "retry_count", "forced",
+    }
+    result = {key: copy.deepcopy(value) for key, value in operation.items() if key in allowed}
+    plan = operation.get("resource_plan") or {}
+    result["resource_plan"] = {
+        "tunnel_disposition": plan.get("tunnel_disposition", "unknown"),
+        "rule_count": len(plan.get("rule_keys") or []),
+        "dns_record_count": len(plan.get("dns_targets") or []),
+        "access_app_count": len(plan.get("access_app_ids") or []),
+    }
+    if operation.get("state") in TERMINAL_STATES:
+        plan = operation.get("host_cleanup_plan") or {}
+        images = [
+            _valid_image_reference(plan.get("agent_image")),
+            "tecnativa/docker-socket-proxy:v0.4.1",
+            "alpine:3.20",
+            _valid_image_reference(plan.get("cloudflared_image")),
+        ]
+        image_args = " ".join(value for value in images if value)
+        result["cleanup_commands"] = {
+            "docker": "\n".join([
+                "#!/usr/bin/env bash",
+                "set -euo pipefail",
+                'DEPLOY_DIR="${HOME:?}/dockflare-agent"',
+                'if docker compose version >/dev/null 2>&1; then COMPOSE_CMD=(docker compose);',
+                'elif docker-compose version >/dev/null 2>&1; then COMPOSE_CMD=(docker-compose);',
+                'else echo "Error: Docker Compose is not available." >&2; exit 1; fi',
+                "docker stop dockflare-agent-tunnel >/dev/null 2>&1 || true",
+                "docker rm dockflare-agent-tunnel >/dev/null 2>&1 || true",
+                'cd "$DEPLOY_DIR"',
+                '"${COMPOSE_CMD[@]}" -f ./docker-compose.yml down --volumes',
+                f"docker image rm {image_args} || true",
+            ]),
+            "deployment_files": 'cd "${HOME:?}" && rm -rf -- ./dockflare-agent',
+        }
+    return result

dockflare/app/core/cloudflare_api.py

@@ -495,6 +495,12 @@ def delete_cloudflare_dns_record(zone_id, hostname, tunnel_id):
         if not record_id:
             logging.warning(f"DNS record for {hostname} in zone {zone_id} (for tunnel {tunnel_id}) not found to delete. Assuming success or already deleted.")
             return True
+        if not is_correct_tunnel:
+            logging.warning(
+                "Preserving DNS record for %s because it does not point to the expected tunnel.",
+                hostname,
+            )
+            return False
        
         logging.info(f"Attempting to delete DNS record for {hostname} in zone {zone_id} (ID: {record_id})")
         endpoint = f"/zones/{zone_id}/dns_records/{record_id}"

dockflare/app/core/docker_handler.py

@@ -26,12 +26,20 @@ from flask import current_app
 
 from app import config, docker_client, cloudflared_agent_state, tunnel_state, publish_state_event
 
-from app.core.state_manager import managed_rules, state_lock, save_state
+from app.core.state_manager import (
+    find_container_rule,
+    managed_rules,
+    mark_rule_tunnel_sync_pending,
+    restore_rule_lifecycle,
+    save_state,
+    state_lock,
+)
 from app.core.tunnel_manager import update_cloudflare_config
 from app.core.cloudflare_api import create_cloudflare_dns_record, get_account_zone_inventory, resolve_account_zone
 from app.core.zone_resolver import ZoneResolutionError, normalize_dns_name
 from app.core.access_manager import handle_access_policy_from_labels
-from app.core.utils import get_rule_key, get_label, normalize_access_group_value
+from app.core.utils import get_rule_key, get_source_rule_key, get_label, normalize_access_group_value
+from app.core import notification_manager
 
 def is_valid_hostname(hostname):
     try:
@@ -80,6 +88,8 @@ def process_container_start(container_obj):
 
         container_id_val = None
         container_name_val = "UnknownContainer"
+        activated_resources = []
+        restored_resources = []
         
         try:
             container_id_val = container_obj.id
@@ -263,7 +273,7 @@ def process_container_start(container_obj):
             dns_targets = {}
             master_tunnel_id = tunnel_state.get("id") if not config.USE_EXTERNAL_CLOUDFLARED else config.EXTERNAL_TUNNEL_ID
             master_tunnel_name = tunnel_state.get("name")
-            zone_inventory = get_account_zone_inventory()
+            zone_inventory = None
 
             if not master_tunnel_name or master_tunnel_name == "dockflare-tunnel":
                 from app.core.cloudflare_api import get_tunnel_name_by_id
@@ -278,6 +288,36 @@ def process_container_start(container_obj):
                 service = config_item["service"]
                 path_from_item = config_item.get("path")
                 rule_key = get_rule_key(hostname, path_from_item)
+                source_rule_key = get_source_rule_key(hostname, path_from_item)
+
+                with state_lock:
+                    matched_key, matched_rule = find_container_rule(source_rule_key, "docker")
+                    if matched_rule and matched_rule.get("rule_ui_override", False):
+                        changed, reactivated = restore_rule_lifecycle(matched_rule, container_id_val)
+                        if matched_rule.get("source_rule_key") is None and matched_key == rule_key:
+                            matched_rule["source_rule_key"] = source_rule_key
+                            matched_rule["lifecycle_generation"] = int(matched_rule.get("lifecycle_generation") or 0) + 1
+                            changed = True
+                        if reactivated:
+                            mark_rule_tunnel_sync_pending(matched_rule)
+                            needs_tunnel_config_update_for_this_container = True
+                            restored_resources.append({
+                                "key": matched_key,
+                                "hostname": matched_rule.get("hostname"),
+                                "path": matched_rule.get("path"),
+                                "source": "docker",
+                            })
+                        state_changed_locally_for_this_container |= changed
+                        if matched_rule.get("hostname") and matched_rule.get("zone_id"):
+                            dns_targets[matched_rule["hostname"]] = {"zone_id": matched_rule["zone_id"]}
+                        outcome = "lifecycle restored" if reactivated else "container association refreshed"
+                        logging.info(
+                            "UI-overridden rule %s: %s for container %s; UI configuration preserved.",
+                            matched_key,
+                            outcome,
+                            container_id_val[:12],
+                        )
+                        continue
                 
                 zone_name_from_item = config_item["zone_name"]
                 no_tls_verify_from_item = config_item["no_tls_verify"]
@@ -287,6 +327,8 @@ def process_container_start(container_obj):
                 disable_chunked_encoding_from_item = config_item.get("disable_chunked_encoding", False)
                 match_sni_to_host_from_item = config_item.get("match_sni_to_host", False)
 
+                if zone_inventory is None:
+                    zone_inventory = get_account_zone_inventory()
                 try:
                     selected_zone = resolve_account_zone(
                         hostname,
@@ -297,6 +339,11 @@ def process_container_start(container_obj):
                     )
                 except ZoneResolutionError as exc:
                     logging.error(f"DOCKER_HANDLER: Zone resolution failed for {rule_key} ({exc.code}). Skipping rule.")
+                    notification_manager.emit(
+                        "cloudflare.dns_failure",
+                        rule_key,
+                        {"hostname": hostname, "path": path_from_item, "operation": "zone resolution", "source": "docker"},
+                    )
                     continue
                 target_zone_id = selected_zone["id"]
                 config_item["zone_name"] = selected_zone.get("name")
@@ -309,6 +356,9 @@ def process_container_start(container_obj):
                     if existing_rule and existing_rule.get("source") == "manual":
                         logging.info(f"DOCKER_HANDLER: Rule {rule_key} is manual, skipping for {container_name_val}.")
                         continue
+                    if existing_rule and existing_rule.get("source", "docker") != "docker":
+                        logging.warning("DOCKER_HANDLER: Rule %s belongs to another source; observation ignored.", rule_key)
+                        continue
 
                     if existing_rule and existing_rule.get("rule_ui_override", False):
                         logging.info(f"DOCKER_HANDLER: Rule {rule_key} is UI-overridden, skipping Docker updates for {container_name_val}.")
@@ -317,6 +367,9 @@ def process_container_start(container_obj):
                     original_existing_rule_for_comparison = copy.deepcopy(existing_rule) if existing_rule else None
                     
                     if existing_rule:
+                        if existing_rule.get("source_rule_key") is None:
+                            existing_rule["source_rule_key"] = source_rule_key
+                            existing_rule["lifecycle_generation"] = int(existing_rule.get("lifecycle_generation") or 0) + 1
                         logging.debug(f"DOCKER_HANDLER_UPD_RULE_PRE: Updating rule for {rule_key}. Current: {existing_rule}")
 
                         rule_data_changed = False
@@ -369,8 +422,24 @@ def process_container_start(container_obj):
                             existing_rule["status"] = "active"
                             existing_rule["delete_at"] = None
                             rule_data_changed = True
+                            restored_resources.append({
+                                "key": rule_key,
+                                "hostname": hostname,
+                                "path": path_from_item,
+                                "source": "docker",
+                            })
 
-                        if rule_data_changed:
+                        tunnel_fields = (
+                            "hostname", "path", "service", "zone_id", "no_tls_verify",
+                            "origin_server_name", "http_host_header", "http2_origin",
+                            "disable_chunked_encoding", "match_sni_to_host", "tunnel_id",
+                        )
+                        requires_tunnel_update = (
+                            original_existing_rule_for_comparison.get("status") != existing_rule.get("status")
+                            or any(original_existing_rule_for_comparison.get(field) != existing_rule.get(field) for field in tunnel_fields)
+                        )
+                        if requires_tunnel_update:
+                            mark_rule_tunnel_sync_pending(existing_rule)
                             needs_tunnel_config_update_for_this_container = True
 
                         if original_existing_rule_for_comparison != existing_rule:
@@ -403,11 +472,22 @@ def process_container_start(container_obj):
                             "source": "docker",
                             "access_group_id": None,
                             "tunnel_id": master_tunnel_id,
-                            "tunnel_name": master_tunnel_name
+                            "tunnel_name": master_tunnel_name,
+                            "source_rule_key": source_rule_key,
+                            "tunnel_sync_pending": True,
+                            "tunnel_sync_last_attempt_at": None,
+                            "tunnel_sync_attempts": 0,
+                            "lifecycle_generation": 0,
                         }
                         existing_rule = managed_rules[rule_key]
                         state_changed_locally_for_this_container = True
                         needs_tunnel_config_update_for_this_container = True
+                        activated_resources.append({
+                            "key": rule_key,
+                            "hostname": hostname,
+                            "path": path_from_item,
+                            "source": "docker",
+                        })
                         logging.debug(f"DOCKER_HANDLER_NEW_RULE_POST: Added {rule_key}. Rule: {existing_rule}")
 
                     dns_targets[hostname] = {
@@ -424,8 +504,21 @@ def process_container_start(container_obj):
 
             policy_state_changed = False
             for rule_key, policy_payload in policy_jobs:
-                if handle_access_policy_from_labels(rule_key, copy.deepcopy(policy_payload)):
-                    policy_state_changed = True
+                try:
+                    if handle_access_policy_from_labels(rule_key, copy.deepcopy(policy_payload)):
+                        policy_state_changed = True
+                except Exception:
+                    notification_manager.emit(
+                        "cloudflare.access_failure",
+                        rule_key,
+                        {
+                            "hostname": policy_payload.get("hostname"),
+                            "path": policy_payload.get("path"),
+                            "operation": "apply policy",
+                            "source": "docker",
+                        },
+                    )
+                    raise
 
             if policy_state_changed:
                 state_changed_locally_for_this_container = True
@@ -439,25 +532,98 @@ def process_container_start(container_obj):
             if needs_tunnel_config_update_for_this_container:
                 logging.info(f"DOCKER_HANDLER: Triggering tunnel config update for {container_name_val}.")
                 if update_cloudflare_config():
+                    remote_success = True
+                    sync_state_changed = False
+                    with state_lock:
+                        for rule in managed_rules.values():
+                            if rule.get("source") == "docker" and rule.get("tunnel_sync_pending"):
+                                rule["tunnel_sync_pending"] = False
+                                rule["tunnel_sync_last_attempt_at"] = None
+                                rule["tunnel_sync_attempts"] = 0
+                                rule["lifecycle_generation"] = int(rule.get("lifecycle_generation") or 0) + 1
+                                sync_state_changed = True
+                    if sync_state_changed:
+                        save_state()
                     logging.info(f"DOCKER_HANDLER: Tunnel config update successful for {container_name_val}.")
                     effective_tunnel_id = tunnel_state.get("id") if not config.USE_EXTERNAL_CLOUDFLARED else config.EXTERNAL_TUNNEL_ID
                     if effective_tunnel_id:
                         for hostname_dns, dns_details in dns_targets.items():
                             target_zone_id_for_dns_item = dns_details.get("zone_id")
                             if target_zone_id_for_dns_item and not hostname_dns.startswith('*.'):
-                                dns_record_id_status = create_cloudflare_dns_record(target_zone_id_for_dns_item, hostname_dns, effective_tunnel_id)
+                                try:
+                                    dns_record_id_status = create_cloudflare_dns_record(
+                                        target_zone_id_for_dns_item, hostname_dns, effective_tunnel_id
+                                    )
+                                except Exception:
+                                    logging.exception("DOCKER_HANDLER: DNS creation raised for %s.", hostname_dns)
+                                    dns_record_id_status = None
                                 if dns_record_id_status and dns_record_id_status not in ["semaphore_timeout", "existing_record_unconfirmed"]:
                                     logging.info(f"DOCKER_HANDLER: DNS for {hostname_dns} in zone {target_zone_id_for_dns_item} OK (ID/Status: {dns_record_id_status}).")
                                 elif not dns_record_id_status:
                                     logging.error(f"DOCKER_HANDLER: CRITICAL - Failed DNS for {hostname_dns} in zone {target_zone_id_for_dns_item}!")
+                                    remote_success = False
+                                    notification_manager.emit(
+                                        "cloudflare.dns_failure",
+                                        f"{target_zone_id_for_dns_item}:{hostname_dns}",
+                                        {"hostname": hostname_dns, "operation": "create", "source": "docker"},
+                                    )
                                     if cloudflared_agent_state:
                                         cloudflared_agent_state["last_action_status"] = f"Error: Failed DNS for {hostname_dns}."
+                                elif dns_record_id_status in ["semaphore_timeout", "existing_record_unconfirmed"]:
+                                    remote_success = False
+                                    notification_manager.emit(
+                                        "cloudflare.dns_failure",
+                                        f"{target_zone_id_for_dns_item}:{hostname_dns}",
+                                        {"hostname": hostname_dns, "operation": "create", "source": "docker"},
+                                    )
                             elif not target_zone_id_for_dns_item:
                                 logging.error(f"DOCKER_HANDLER: No Zone ID for DNS for {hostname_dns} - cannot manage record.")
+                                remote_success = False
+                                notification_manager.emit(
+                                    "cloudflare.dns_failure",
+                                    hostname_dns,
+                                    {"hostname": hostname_dns, "operation": "create", "source": "docker"},
+                                )
                     else:
                         logging.error(f"DOCKER_HANDLER: Missing effective Tunnel ID - cannot manage DNS records for {container_name_val}.")
+                        remote_success = False
+                        notification_manager.emit(
+                            "cloudflare.tunnel_failure",
+                            container_id_val,
+                            {"container_name": container_name_val, "container_id": container_id_val[:12], "operation": "resolve tunnel", "source": "docker"},
+                        )
+                    if remote_success:
+                        common_context = {
+                            "container_name": container_name_val,
+                            "container_id": container_id_val[:12],
+                            "source": "docker",
+                            "public_url": config.DOCKFLARE_PUBLIC_URL,
+                        }
+                        if activated_resources:
+                            notification_manager.emit(
+                                "rule.activated",
+                                container_id_val,
+                                {**common_context, "resources": activated_resources},
+                            )
+                        if restored_resources:
+                            notification_manager.emit(
+                                "rule.restored",
+                                container_id_val,
+                                {**common_context, "resources": restored_resources},
+                            )
                 else:
                     logging.error(f"DOCKER_HANDLER: Failed to update Cloudflare tunnel config for {container_name_val}. DNS records not managed.")
+                    notification_manager.emit(
+                        "cloudflare.tunnel_failure",
+                        str(tunnel_state.get("id") or config.EXTERNAL_TUNNEL_ID or container_id_val),
+                        {
+                            "container_name": container_name_val,
+                            "container_id": container_id_val[:12],
+                            "tunnel_id": str(tunnel_state.get("id") or config.EXTERNAL_TUNNEL_ID or "")[:12],
+                            "operation": "update",
+                            "source": "docker",
+                        },
+                    )
 
 
         except NotFound:
@@ -478,6 +644,8 @@ def schedule_container_stop(container_id_val):
         logging.info(f"Processing stop event for container {container_id_val[:12]}.")
         
         state_changed_after_stop_processing = False
+        pending_resources = []
+        delete_at_value = None
         with state_lock:
             rule_keys_affected_by_stop = []
             for r_key, details in managed_rules.items():
@@ -494,7 +662,15 @@ def schedule_container_stop(container_id_val):
                         rule["status"] = "pending_deletion"
                         grace_delta = timedelta(seconds=grace_period)
                         rule["delete_at"] = datetime.now(timezone.utc) + grace_delta
+                        rule["lifecycle_generation"] = int(rule.get("lifecycle_generation") or 0) + 1
                         logging.info(f"Rule for {rule_key_to_schedule} (from stopped container {container_id_val[:12]}) scheduled for deletion at {rule['delete_at'].isoformat()}")
+                        delete_at_value = rule['delete_at'].isoformat()
+                        pending_resources.append({
+                            "key": rule_key_to_schedule,
+                            "hostname": rule.get("hostname"),
+                            "path": rule.get("path"),
+                            "source": "docker",
+                        })
                         state_changed_after_stop_processing = True
                     else:
                         logging.info(f"Rule for {rule_key_to_schedule} from stopped container {container_id_val[:12]} was already pending deletion.")
@@ -504,6 +680,19 @@ def schedule_container_stop(container_id_val):
             if state_changed_after_stop_processing:
                 save_state()
                 publish_state_event('snapshot_refresh')
+        if pending_resources:
+            notification_manager.emit(
+                "rule.pending_deletion",
+                container_id_val,
+                {
+                    "container_id": container_id_val[:12],
+                    "source": "docker",
+                    "delete_at": delete_at_value,
+                    "grace_period_seconds": current_app.config.get('GRACE_PERIOD_SECONDS', 28800),
+                    "resources": pending_resources,
+                    "public_url": config.DOCKFLARE_PUBLIC_URL,
+                },
+            )
 
 def docker_event_listener(stop_event_param, label_prefix):
     if not docker_client:
@@ -580,6 +769,11 @@ def docker_event_listener(stop_event_param, label_prefix):
 
     if error_count >= max_errors:
         logging.error(f"Docker event listener for {label_prefix} stopping after multiple consecutive errors.")
+        notification_manager.emit(
+            "docker.listener_failure",
+            label_prefix,
+            {"message": "The Docker event listener stopped after repeated connection failures."},
+        )
     logging.info(f"Docker event listener for {label_prefix} stopped.")
 
 def start_event_listeners(stop_event):

dockflare/app/core/notification_manager.py

@@ -0,0 +1,794 @@
+import copy
+import itertools
+import logging
+import os
+import queue
+import threading
+import time
+import uuid
+from datetime import datetime, timezone
+from urllib.parse import urlsplit, urlunsplit
+
+
+EVENT_DEFINITIONS = {
+    "rule.activated": ("rule_activated", "success", False),
+    "rule.restored": ("rule_restored", "success", False),
+    "rule.pending_deletion": ("rule_pending_deletion", "warning", False),
+    "rule.deleted": ("rule_deleted", "info", True),
+    "cloudflare.tunnel_failure": ("cloudflare_tunnel_failure", "failure", True),
+    "cloudflare.dns_failure": ("cloudflare_dns_failure", "failure", True),
+    "cloudflare.access_failure": ("cloudflare_access_failure", "failure", True),
+    "docker.listener_failure": ("docker_listener_failure", "failure", True),
+    "agent.offline": ("agent_offline", "failure", True),
+    "agent.online": ("agent_online", "success", True),
+    "agent.enrolled": ("agent_enrolled", "success", True),
+    "agent.enrollment_failed": ("agent_enrollment_failed", "failure", True),
+    "agent.decommission_started": ("agent_decommission_started", "warning", False),
+    "agent.decommission_completed": ("agent_decommission_completed", "success", True),
+    "agent.decommission_failed": ("agent_decommission_failed", "failure", True),
+    "agent.decommission_stalled": ("agent_decommission_stalled", "warning", True),
+    "tunnel.down": ("tunnel_down", "failure", True),
+    "tunnel.recovered": ("tunnel_recovered", "success", True),
+    "access.policy_created": ("access_policy_created", "success", False),
+    "access.policy_updated": ("access_policy_updated", "info", False),
+    "access.policy_deleted": ("access_policy_deleted", "info", False),
+}
+
+DEFAULT_NOTIFICATION_CONFIG = {
+    "enabled": False,
+    "urls": [],
+    "events": {definition[0]: definition[2] for definition in EVENT_DEFINITIONS.values()},
+    "failure_cooldown_seconds": 900,
+}
+
+BOOTSTRAP_SUPPRESSED_EVENTS = {
+    "rule.activated",
+    "rule.restored",
+    "rule.pending_deletion",
+    "rule.deleted",
+    "agent.online",
+    "agent.enrolled",
+    "agent.decommission_started",
+    "agent.decommission_completed",
+    "tunnel.recovered",
+    "access.policy_created",
+    "access.policy_updated",
+    "access.policy_deleted",
+}
+
+COOLDOWN_EVENTS = {
+    "cloudflare.tunnel_failure",
+    "cloudflare.dns_failure",
+    "cloudflare.access_failure",
+    "docker.listener_failure",
+    "agent.offline",
+    "agent.enrollment_failed",
+    "agent.decommission_failed",
+    "agent.decommission_stalled",
+    "tunnel.down",
+}
+
+RECOVERY_PAIRS = {
+    "agent.online": "agent.offline",
+    "tunnel.recovered": "tunnel.down",
+}
+
+TITLE_MAP = {
+    "rule.activated": "✅ DockFlare — Rule activated",
+    "rule.restored": "✅ DockFlare — Rule restored",
+    "rule.pending_deletion": "⚠️ DockFlare — Rule scheduled for deletion",
+    "rule.deleted": "ℹ️ DockFlare — Rule deleted",
+    "cloudflare.tunnel_failure": "❌ DockFlare — Tunnel update failed",
+    "cloudflare.dns_failure": "❌ DockFlare — DNS operation failed",
+    "cloudflare.access_failure": "❌ DockFlare — Access operation failed",
+    "docker.listener_failure": "❌ DockFlare — Docker listener stopped",
+    "agent.offline": "🔴 DockFlare — Agent offline",
+    "agent.online": "🟢 DockFlare — Agent recovered",
+    "agent.enrolled": "✅ DockFlare — Agent enrolled",
+    "agent.enrollment_failed": "❌ DockFlare — Agent enrollment failed",
+    "agent.decommission_started": "⚠️ DockFlare — Agent decommission started",
+    "agent.decommission_completed": "✅ DockFlare — Agent decommission completed",
+    "agent.decommission_failed": "❌ DockFlare — Agent decommission failed",
+    "agent.decommission_stalled": "⚠️ DockFlare — Agent decommission stalled",
+    "tunnel.down": "🔴 DockFlare — Tunnel down",
+    "tunnel.recovered": "🟢 DockFlare — Tunnel recovered",
+    "access.policy_created": "✅ DockFlare — Access Policy created",
+    "access.policy_updated": "ℹ️ DockFlare — Access Policy updated",
+    "access.policy_deleted": "ℹ️ DockFlare — Access Policy deleted",
+    "notification.test": "🔔 DockFlare — Test notification",
+}
+
+STATUS_MAP = {
+    "agent.offline": "Offline",
+    "agent.online": "Online",
+    "tunnel.down": "Down",
+    "tunnel.recovered": "Running",
+    "agent.enrolled": "Enrolled",
+    "agent.enrollment_failed": "Enrollment failed",
+    "agent.decommission_started": "In progress",
+    "agent.decommission_completed": "Completed",
+    "agent.decommission_failed": "Failed",
+    "agent.decommission_stalled": "Timed out",
+    "access.policy_created": "Created",
+    "access.policy_updated": "Updated",
+    "access.policy_deleted": "Deleted",
+}
+
+
+def normalize_notification_config(value):
+    normalized = copy.deepcopy(DEFAULT_NOTIFICATION_CONFIG)
+    if not isinstance(value, dict):
+        return normalized
+
+    normalized["enabled"] = bool(value.get("enabled", False))
+    urls = value.get("urls", [])
+    if isinstance(urls, list):
+        seen = set()
+        valid_shape = []
+        for item in urls:
+            if not isinstance(item, str):
+                continue
+            item = item.strip()
+            if not item or len(item.encode("utf-8")) > 4096 or item in seen:
+                continue
+            seen.add(item)
+            valid_shape.append(item)
+            if len(valid_shape) == 32:
+                break
+        normalized["urls"] = valid_shape
+
+    supplied_events = value.get("events", {})
+    if isinstance(supplied_events, dict):
+        for event_key in normalized["events"]:
+            if event_key in supplied_events:
+                normalized["events"][event_key] = bool(supplied_events[event_key])
+
+    cooldown = value.get("failure_cooldown_seconds", 900)
+    if isinstance(cooldown, int) and not isinstance(cooldown, bool) and 60 <= cooldown <= 86400:
+        normalized["failure_cooldown_seconds"] = cooldown
+    return normalized
+
+
+def sanitize_service(value):
+    if not isinstance(value, str) or not value.strip():
+        return None
+    value = value.strip()
+    try:
+        parsed = urlsplit(value)
+    except ValueError:
+        return None
+    if not parsed.scheme:
+        return value[:256] if all(char not in value for char in ("?", "#", "@")) else None
+    try:
+        hostname = parsed.hostname
+        port_value = parsed.port
+    except ValueError:
+        return parsed.scheme[:32] if parsed.scheme else None
+    if not hostname:
+        return parsed.scheme
+    port = f":{port_value}" if port_value else ""
+    return urlunsplit((parsed.scheme, f"{hostname}{port}", parsed.path or "", "", ""))[:256]
+
+
+def redact_destination(url):
+    if not isinstance(url, str):
+        return "configured destination"
+    try:
+        scheme = urlsplit(url.strip()).scheme.lower()
+    except ValueError:
+        scheme = ""
+    if not scheme or not scheme.replace("+", "").replace("-", "").isalnum():
+        return "configured destination"
+    return f"{scheme}://configured destination"
+
+
+def public_resource_url(hostname, path=None):
+    """Return a safe HTTPS URL for a public rule hostname, if it is linkable."""
+    if not isinstance(hostname, str):
+        return None
+    hostname = hostname.strip()
+    if not hostname or hostname.startswith("*.") or any(char in hostname for char in ("@", "?", "#", "/")):
+        return None
+    try:
+        parsed = urlsplit(f"https://{hostname}")
+        if not parsed.hostname or parsed.username or parsed.password:
+            return None
+        # Reject values that urlsplit normalized into something other than a host[:port].
+        if parsed.netloc.lower() != hostname.lower():
+            return None
+        if parsed.port is not None and not 1 <= parsed.port <= 65535:
+            return None
+    except (TypeError, ValueError):
+        return None
+    safe_path = str(path or "").strip()
+    if safe_path and not safe_path.startswith("/"):
+        safe_path = f"/{safe_path}"
+    if any(char in safe_path for char in ("?", "#", "\r", "\n")):
+        return None
+    return urlunsplit(("https", hostname, safe_path, "", ""))[:768]
+
+
+class NotificationManager:
+    def __init__(self, queue_size=100, max_body_chars=4000, resource_limit=10):
+        self._queue_size = max(1, int(queue_size))
+        self._max_body_chars = max(256, int(max_body_chars))
+        self._resource_limit = max(1, int(resource_limit))
+        self._queue = queue.PriorityQueue(maxsize=self._queue_size)
+        self._sequence = itertools.count()
+        self._lock = threading.RLock()
+        self._config = normalize_notification_config(None)
+        self._apprise = None
+        self._available = False
+        self._invalid_destination_count = 0
+        self._stop_event = None
+        self._worker = None
+        self._accepting = True
+        self._bootstrap = True
+        self._cooldowns = {}
+        self._outages_emitted = set()
+        self._test_jobs = {}
+        self._dropped_events = 0
+        self._last_drop_log_at = 0.0
+        self._last_attempt_at = None
+        self._last_success_at = None
+        self._last_failure_at = None
+        self._agent_health = {}
+        self._tunnel_health = {}
+        self._intentional_tunnels = {}
+
+    @staticmethod
+    def _load_apprise():
+        import apprise
+        return apprise
+
+    def _build_apprise(self, urls, reject_invalid=False):
+        try:
+            apprise_module = self._load_apprise()
+            asset = apprise_module.AppriseAsset(
+                app_id="DockFlare",
+                app_desc="DockFlare Notifications",
+                app_url="https://dockflare.app",
+                secure_logging=True,
+            )
+            instance = apprise_module.Apprise(asset=asset)
+        except Exception as exc:
+            logging.error("NOTIFY_CONFIG: Apprise unavailable (%s).", type(exc).__name__)
+            return None, [(0, "unavailable")]
+
+        invalid = []
+        for index, url in enumerate(urls, 1):
+            try:
+                if not instance.add(url):
+                    invalid.append((index, self._safe_scheme(url)))
+            except Exception:
+                invalid.append((index, self._safe_scheme(url)))
+        if reject_invalid and invalid:
+            return None, invalid
+        return instance if len(instance) else None, invalid
+
+    @staticmethod
+    def _safe_scheme(url):
+        try:
+            scheme = urlsplit(url).scheme.lower()
+        except (TypeError, ValueError):
+            return "unknown"
+        return scheme if scheme and scheme.replace("+", "").replace("-", "").isalnum() else "unknown"
+
+    def validate_urls(self, urls):
+        if not isinstance(urls, list) or len(urls) > 32:
+            return False, [(0, "invalid")]
+        for index, url in enumerate(urls, 1):
+            if not isinstance(url, str) or not url.strip() or len(url.strip().encode("utf-8")) > 4096:
+                return False, [(index, self._safe_scheme(url))]
+        normalized = normalize_notification_config({"urls": urls})["urls"]
+        _instance, invalid = self._build_apprise(normalized, reject_invalid=True)
+        return not invalid, invalid
+
+    def configure(self, value):
+        normalized = normalize_notification_config(value)
+        instance = None
+        invalid = []
+        if normalized["urls"]:
+            instance, invalid = self._build_apprise(normalized["urls"])
+        with self._lock:
+            self._config = normalized
+            self._apprise = instance
+            self._invalid_destination_count = len(invalid)
+            self._available = instance is not None and len(instance) > 0
+        if invalid:
+            logging.warning("NOTIFY_CONFIG: Skipped %s invalid destination(s).", len(invalid))
+        logging.info(
+            "NOTIFY_CONFIG: enabled=%s configured=%s valid=%s",
+            normalized["enabled"],
+            len(normalized["urls"]),
+            len(instance) if instance is not None else 0,
+        )
+        return self.get_public_status()
+
+    def start(self, stop_event):
+        with self._lock:
+            if self._worker and self._worker.is_alive():
+                return self._worker
+            self._stop_event = stop_event
+            self._accepting = True
+            self._worker = threading.Thread(target=self._run, name="NotificationWorker", daemon=True)
+            self._worker.start()
+            logging.info("NOTIFY_WORKER: Started.")
+            return self._worker
+
+    def stop(self, timeout_seconds=5):
+        with self._lock:
+            self._accepting = False
+            worker = self._worker
+        try:
+            self._queue.put_nowait((-1, next(self._sequence), None))
+        except queue.Full:
+            pass
+        if worker and worker.is_alive():
+            worker.join(timeout=max(0, timeout_seconds))
+
+    def begin_bootstrap(self):
+        with self._lock:
+            self._bootstrap = True
+
+    def end_bootstrap(self):
+        with self._lock:
+            self._bootstrap = False
+
+    def emit(self, event_type, resource_id, context=None, notify_type=None):
+        return self._enqueue(event_type, resource_id, context, notify_type, False, None, 10)
+
+    def send_test(self):
+        job_id = str(uuid.uuid4())
+        with self._lock:
+            self._expire_test_jobs_locked()
+            self._test_jobs[job_id] = {
+                "status": "pending",
+                "created_at": time.time(),
+                "completed_at": None,
+            }
+        accepted = self._enqueue(
+            "notification.test",
+            "test",
+            {"message": "DockFlare notification delivery is configured correctly."},
+            "info",
+            True,
+            job_id,
+            0,
+        )
+        if not accepted:
+            with self._lock:
+                self._test_jobs[job_id].update({"status": "failure", "completed_at": self._utc_now()})
+        return job_id, accepted
+
+    def get_test_status(self, job_id):
+        with self._lock:
+            self._expire_test_jobs_locked()
+            value = self._test_jobs.get(job_id)
+            return copy.deepcopy(value) if value else None
+
+    def _expire_test_jobs_locked(self):
+        cutoff = time.time() - 600
+        for job_id in [key for key, value in self._test_jobs.items() if value["created_at"] < cutoff]:
+            self._test_jobs.pop(job_id, None)
+
+    def _enqueue(self, event_type, resource_id, context, notify_type, is_test, job_id, priority):
+        if event_type != "notification.test" and event_type not in EVENT_DEFINITIONS:
+            return False
+        safe_resource = str(resource_id or "unknown")[:512]
+        now = time.monotonic()
+        cooldown_key = None
+        outage_key = None
+        with self._lock:
+            config_snapshot = copy.deepcopy(self._config)
+            if not self._accepting or not config_snapshot["enabled"] or not self._available:
+                return False
+            if not is_test:
+                setting_key, default_type, _default_enabled = EVENT_DEFINITIONS[event_type]
+                if not config_snapshot["events"].get(setting_key, False):
+                    return False
+                if self._bootstrap and event_type in BOOTSTRAP_SUPPRESSED_EVENTS:
+                    logging.info("NOTIFY_SUPPRESSED: %s during bootstrap.", event_type)
+                    return False
+                notify_type = notify_type or default_type
+                cooldown_key = f"{event_type}:{safe_resource}"
+                if event_type in COOLDOWN_EVENTS:
+                    last = self._cooldowns.get(cooldown_key)
+                    if last is not None and now - last < config_snapshot["failure_cooldown_seconds"]:
+                        logging.info("NOTIFY_SUPPRESSED: %s cooldown active.", event_type)
+                        return False
+                recovery_for = RECOVERY_PAIRS.get(event_type)
+                if recovery_for:
+                    outage_key = f"{recovery_for}:{safe_resource}"
+                    if outage_key not in self._outages_emitted:
+                        return False
+
+        event = {
+            "event_id": str(uuid.uuid4()),
+            "event_type": event_type,
+            "resource_id": safe_resource,
+            "occurred_at": self._utc_now(),
+            "notify_type": notify_type or "info",
+            "context": self._sanitize_context(context or {}),
+            "is_test": is_test,
+            "job_id": job_id,
+        }
+        try:
+            self._queue.put_nowait((priority, next(self._sequence), event))
+            with self._lock:
+                if cooldown_key and event_type in COOLDOWN_EVENTS:
+                    self._cooldowns[cooldown_key] = now
+                    self._outages_emitted.add(cooldown_key)
+                if outage_key:
+                    self._outages_emitted.discard(outage_key)
+                    self._cooldowns.pop(outage_key, None)
+            logging.info("NOTIFY_QUEUED: %s event_id=%s", event_type, event["event_id"])
+            return True
+        except queue.Full:
+            with self._lock:
+                self._dropped_events += 1
+                if now - self._last_drop_log_at >= 60:
+                    logging.warning("NOTIFY_DROPPED: Notification queue is full.")
+                    self._last_drop_log_at = now
+            return False
+
+    def _run(self):
+        while True:
+            stop_requested = self._stop_event is not None and self._stop_event.is_set()
+            if stop_requested and self._queue.empty():
+                break
+            try:
+                _priority, _sequence, event = self._queue.get(timeout=0.5)
+            except queue.Empty:
+                continue
+            if event is None:
+                self._queue.task_done()
+                if stop_requested or not self._accepting:
+                    break
+                continue
+            try:
+                self._deliver(event)
+            except Exception as exc:
+                logging.error(
+                    "NOTIFY_FAILED: event_type=%s event_id=%s error=%s",
+                    event["event_type"], event["event_id"], type(exc).__name__,
+                )
+                self._record_delivery(event, False)
+            finally:
+                self._queue.task_done()
+        logging.info("NOTIFY_WORKER: Stopped.")
+
+    def _deliver(self, event):
+        with self._lock:
+            instance = self._apprise
+        if instance is None:
+            self._record_delivery(event, False)
+            return
+        title, body = self._render(event)
+        notify_type = self._resolve_notify_type(event["notify_type"])
+        started = time.monotonic()
+        success = bool(instance.notify(title=title, body=body, notify_type=notify_type))
+        duration = time.monotonic() - started
+        self._record_delivery(event, success)
+        log_method = logging.info if success else logging.error
+        log_method(
+            "%s: event_type=%s event_id=%s destinations=%s duration=%.2fs",
+            "NOTIFY_DELIVERED" if success else "NOTIFY_FAILED",
+            event["event_type"],
+            event["event_id"],
+            len(instance),
+            duration,
+        )
+
+    def _resolve_notify_type(self, value):
+        try:
+            apprise_module = self._load_apprise()
+            return {
+                "info": apprise_module.NotifyType.INFO,
+                "success": apprise_module.NotifyType.SUCCESS,
+                "warning": apprise_module.NotifyType.WARNING,
+                "failure": apprise_module.NotifyType.FAILURE,
+            }.get(value, apprise_module.NotifyType.INFO)
+        except Exception:
+            return value
+
+    def _record_delivery(self, event, success):
+        completed_at = self._utc_now()
+        with self._lock:
+            self._last_attempt_at = completed_at
+            if success:
+                self._last_success_at = completed_at
+            else:
+                self._last_failure_at = completed_at
+            job_id = event.get("job_id")
+            if job_id and job_id in self._test_jobs:
+                self._test_jobs[job_id].update({
+                    "status": "success" if success else "failure",
+                    "completed_at": completed_at,
+                })
+
+    def _render(self, event):
+        event_type = event["event_type"]
+        context = event["context"]
+        title = TITLE_MAP.get(event_type, "🔔 DockFlare — Notification")
+        resources = context.get("resources") if isinstance(context.get("resources"), list) else []
+        if len(resources) > 1 and event_type.startswith("rule."):
+            title = title.replace("Rule ", f"{len(resources)} rules ")
+
+        lines = []
+        message = context.get("message")
+        if message:
+            lines.append(str(message))
+
+        if resources:
+            rendered_resources = []
+            for resource in resources[: self._resource_limit]:
+                if not isinstance(resource, dict):
+                    continue
+                hostname = str(resource.get("hostname") or resource.get("key") or "resource")[:255]
+                path = str(resource.get("path") or "")[:255]
+                clickable_url = public_resource_url(hostname, path)
+                rendered_resources.append(clickable_url or f"{hostname}{path}")
+            if len(rendered_resources) == 1:
+                lines.append(f"Service: {rendered_resources[0]}")
+            elif rendered_resources:
+                lines.append("Affected services:")
+                lines.extend(f"- {resource}" for resource in rendered_resources)
+            omitted = len(resources) - self._resource_limit
+            if omitted > 0:
+                lines.append(f"- ... and {omitted} more")
+
+        if not resources and context.get("hostname"):
+            service_url = public_resource_url(context["hostname"], context.get("path"))
+            if service_url:
+                lines.append(f"Service: {service_url}")
+            else:
+                lines.append(f"Hostname: {context['hostname']}")
+
+        if context.get("operation"):
+            lines.append(f"Operation: {context['operation']}")
+        if event_type in STATUS_MAP:
+            lines.append(f"Status: {STATUS_MAP[event_type]}")
+        if context.get("source"):
+            source = str(context["source"])
+            lines.append(f"Source: {source[:1].upper()}{source[1:]}")
+        for key, label in (("container_name", "Container"), ("agent_name", "Agent"), ("tunnel_name", "Tunnel")):
+            if context.get(key) not in (None, ""):
+                lines.append(f"{label}: {context[key]}")
+
+        if context.get("policy_name"):
+            lines.append(f"Access Policy: {context['policy_name']}")
+        if context.get("policy_type"):
+            lines.append(f"Policy type: {context['policy_type']}")
+        if context.get("identity_provider_count"):
+            lines.append(f"Identity providers: {context['identity_provider_count']}")
+        if context.get("rules_count"):
+            rules_label = "Managed rules" if event_type.startswith("agent.decommission_") else "Policy rules"
+            lines.append(f"{rules_label}: {context['rules_count']}")
+        if context.get("affected_service_count"):
+            lines.append(f"Affected services: {context['affected_service_count']}")
+
+        if context.get("delete_at"):
+            lines.append(f"Deletion deadline: {context['delete_at']}")
+        if context.get("grace_period_seconds"):
+            lines.append(f"Grace period: {context['grace_period_seconds']} seconds")
+        if context.get("retry_count"):
+            lines.append(f"Retry count: {context['retry_count']}")
+        if context.get("service"):
+            origin = sanitize_service(context["service"])
+            if origin:
+                lines.append(f"Origin: {origin}")
+
+        technical_fields = [
+            (label, context[key])
+            for key, label in (
+                ("container_id", "Container ID"), ("agent_id", "Agent ID"),
+                ("tunnel_id", "Tunnel ID"), ("operation_id", "Operation ID"),
+                ("policy_id", "Policy ID"),
+            )
+            if context.get(key) not in (None, "")
+        ]
+        if technical_fields:
+            if lines and lines[-1] != "":
+                lines.append("")
+            lines.append("Technical details")
+            lines.extend(f"{label}: {value}" for label, value in technical_fields)
+
+        if context.get("public_url"):
+            public_url = sanitize_service(context["public_url"])
+            if public_url:
+                if lines and lines[-1] != "":
+                    lines.append("")
+                lines.append(f"Dashboard: {public_url}")
+        if not lines:
+            lines.append(f"Event: {event_type}")
+        body = "\n".join(lines)
+        if len(body) > self._max_body_chars:
+            body = body[: self._max_body_chars - 16].rstrip() + "\n[truncated]"
+        return title, body
+
+    def _sanitize_context(self, context):
+        allowed = {
+            "message", "operation", "hostname", "path", "service", "source",
+            "container_name", "container_id", "agent_name", "agent_id",
+            "tunnel_name", "tunnel_id", "delete_at", "grace_period_seconds",
+            "retry_count", "resources", "public_url", "operation_id",
+            "policy_name", "policy_id", "policy_type", "identity_provider_count",
+            "rules_count", "affected_service_count",
+        }
+        sanitized = {}
+        for key in allowed:
+            value = context.get(key)
+            if value is None:
+                continue
+            if key == "resources" and isinstance(value, list):
+                sanitized[key] = [
+                    {
+                        field: str(item[field])[:512]
+                        for field in ("key", "hostname", "path", "source")
+                        if isinstance(item, dict) and field in item and item[field] is not None
+                    }
+                    for item in value[:1000]
+                    if isinstance(item, dict)
+                ]
+            elif isinstance(value, (str, int, float, bool)):
+                sanitized[key] = str(value)[:2048]
+        return sanitized
+
+    def get_public_status(self):
+        with self._lock:
+            self._expire_test_jobs_locked()
+            urls = list(self._config["urls"])
+            return {
+                "enabled": self._config["enabled"],
+                "available": self._available,
+                "configured_destination_count": len(urls),
+                "valid_destination_count": max(0, len(urls) - self._invalid_destination_count),
+                "destinations": [
+                    {"index": index, "scheme": self._safe_scheme(url), "summary": redact_destination(url)}
+                    for index, url in enumerate(urls, 1)
+                ],
+                "events": copy.deepcopy(self._config["events"]),
+                "failure_cooldown_seconds": self._config["failure_cooldown_seconds"],
+                "queue_depth": self._queue.qsize(),
+                "queue_capacity": self._queue_size,
+                "dropped_events": self._dropped_events,
+                "last_attempt_at": self._last_attempt_at,
+                "last_success_at": self._last_success_at,
+                "last_failure_at": self._last_failure_at,
+                "bootstrap": self._bootstrap,
+            }
+
+    def check_agent_health(self, agents_snapshot, heartbeat_timeout, now=None, decommissioning_ids=None):
+        now = now or datetime.now(timezone.utc)
+        decommissioning_ids = set(decommissioning_ids or [])
+        for agent_id, agent in (agents_snapshot or {}).items():
+            if not isinstance(agent, dict) or agent.get("status") != "enrolled" or agent_id in decommissioning_ids:
+                continue
+            online = False
+            try:
+                last_seen = datetime.fromisoformat(str(agent.get("last_seen", "")).replace("Z", "+00:00"))
+                if last_seen.tzinfo is None:
+                    last_seen = last_seen.replace(tzinfo=timezone.utc)
+                online = (now - last_seen.astimezone(timezone.utc)).total_seconds() <= heartbeat_timeout
+            except (TypeError, ValueError):
+                online = False
+            previous = self._agent_health.get(agent_id)
+            self._agent_health[agent_id] = online
+            if previous is None:
+                continue
+            context = {
+                "agent_name": agent.get("display_name") or f"agent-{agent_id[:8]}",
+                "agent_id": agent_id[:12],
+            }
+            if previous and not online:
+                self.emit("agent.offline", agent_id, context)
+            elif not previous and online:
+                self.emit("agent.online", agent_id, context)
+
+    def check_tunnel_health(self, tunnel_id, status, context=None, intentional=False):
+        if not tunnel_id:
+            return
+        now = time.monotonic()
+        with self._lock:
+            intentional_until = self._intentional_tunnels.get(tunnel_id, 0)
+            if intentional_until and intentional_until <= now:
+                self._intentional_tunnels.pop(tunnel_id, None)
+                intentional_until = 0
+        intentional = intentional or intentional_until > now
+        healthy = str(status or "").lower() == "running"
+        state = self._tunnel_health.setdefault(tunnel_id, {"healthy": None, "unhealthy_checks": 0})
+        previous = state["healthy"]
+        if healthy:
+            state["healthy"] = True
+            state["unhealthy_checks"] = 0
+            if previous is False:
+                self.emit("tunnel.recovered", tunnel_id, context or {})
+            return
+        if intentional:
+            state["healthy"] = False
+            state["unhealthy_checks"] = 0
+            return
+        state["unhealthy_checks"] += 1
+        if previous is None:
+            state["healthy"] = False
+            return
+        if previous is True and state["unhealthy_checks"] >= 2:
+            state["healthy"] = False
+            self.emit("tunnel.down", tunnel_id, context or {})
+
+    def suppress_tunnel_health(self, tunnel_id, seconds=120):
+        if not tunnel_id:
+            return
+        with self._lock:
+            self._intentional_tunnels[tunnel_id] = time.monotonic() + max(1, int(seconds))
+
+    @staticmethod
+    def _utc_now():
+        return datetime.now(timezone.utc).isoformat()
+
+
+def _runtime_int(name, default, minimum):
+    try:
+        return max(minimum, int(os.getenv(name, str(default))))
+    except (TypeError, ValueError):
+        return default
+
+
+notification_manager = NotificationManager(
+    queue_size=_runtime_int("NOTIFICATION_QUEUE_SIZE", 100, 1),
+    max_body_chars=_runtime_int("NOTIFICATION_MAX_BODY_CHARS", 4000, 256),
+    resource_limit=_runtime_int("NOTIFICATION_RESOURCE_LIST_LIMIT", 10, 1),
+)
+
+
+def configure(value):
+    return notification_manager.configure(value)
+
+
+def start(stop_event):
+    return notification_manager.start(stop_event)
+
+
+def stop(timeout_seconds=5):
+    return notification_manager.stop(timeout_seconds)
+
+
+def begin_bootstrap():
+    return notification_manager.begin_bootstrap()
+
+
+def end_bootstrap():
+    return notification_manager.end_bootstrap()
+
+
+def emit(event_type, resource_id, context=None, notify_type=None):
+    return notification_manager.emit(event_type, resource_id, context, notify_type)
+
+
+def send_test():
+    return notification_manager.send_test()
+
+
+def get_test_status(job_id):
+    return notification_manager.get_test_status(job_id)
+
+
+def get_public_status():
+    return notification_manager.get_public_status()
+
+
+def validate_urls(urls):
+    return notification_manager.validate_urls(urls)
+
+
+def check_agent_health(agents_snapshot, heartbeat_timeout, now=None, decommissioning_ids=None):
+    return notification_manager.check_agent_health(agents_snapshot, heartbeat_timeout, now, decommissioning_ids)
+
+
+def check_tunnel_health(tunnel_id, status, context=None, intentional=False):
+    return notification_manager.check_tunnel_health(tunnel_id, status, context, intentional)
+
+
+def suppress_tunnel_health(tunnel_id, seconds=120):
+    return notification_manager.suppress_tunnel_health(tunnel_id, seconds)

dockflare/app/core/reconciler.py

@@ -23,7 +23,17 @@ from datetime import datetime, timedelta, timezone
 from app import config, docker_client, tunnel_state, publish_state_event
 from flask import current_app 
 
-from app.core.state_manager import managed_rules, state_lock, save_state, get_agent, update_agent
+from app.core.state_manager import (
+    find_container_rule,
+    managed_rules,
+    mark_rule_tunnel_sync_pending,
+    restore_container_rule_key,
+    restore_rule_lifecycle,
+    state_lock,
+    save_state,
+    get_agent,
+    update_agent,
+)
 from app.core.cloudflare_api import (
     get_account_zone_inventory,
     resolve_account_zone,
@@ -35,8 +45,9 @@ from app.core.access_manager import (
     handle_access_policy_from_labels, 
     delete_cloudflare_access_application 
 )
-from app.core.tunnel_manager import update_cloudflare_config
-from app.core.utils import get_rule_key, get_label
+from app.core.tunnel_manager import get_tunnel_operation_lock, update_cloudflare_config
+from app.core.utils import get_rule_key, get_source_rule_key, get_label
+from app.core import notification_manager
 
 def _get_hostname_configs_from_container(container_obj):
     labels = container_obj.labels
@@ -195,8 +206,7 @@ def reconcile_agent_report(agent_id, reported_containers):
                     last_auto_dt = last_auto_dt.replace(tzinfo=timezone.utc) if last_auto_dt.tzinfo is None else last_auto_dt.astimezone(timezone.utc)
                     elapsed = (now_dt - last_auto_dt).total_seconds()
                     if elapsed < cooldown:
-                        logging.info(f"[Reconcile-Agent] Auto-restore cooldown active for agent {agent_id} ({elapsed:.1f}s < {cooldown}s). Skipping.")
-                        return False
+                        logging.debug(f"[Reconcile-Agent] Cooldown active for agent {agent_id}; current observations will still be evaluated.")
                 except Exception:
                     logging.debug(f"[Reconcile-Agent] Could not parse last_auto_restore_at for agent {agent_id}; continuing.")
             
@@ -205,7 +215,9 @@ def reconcile_agent_report(agent_id, reported_containers):
                 return False
 
             restored_any = False
+            sync_needed = False
             desired_configs = {}
+            dns_rekeys = []
 
             for c in reported_containers:
                 labels = c.get("labels", {}) or {}
@@ -228,9 +240,28 @@ def reconcile_agent_report(agent_id, reported_containers):
                 logging.debug(f"[Reconcile-Agent] No hostname configs derived from agent {agent_id} reported containers.")
                 return False
 
-            zone_inventory = get_account_zone_inventory()
+            overridden_keys = set()
+            with state_lock:
+                for observed_key, desired in desired_configs.items():
+                    source_key = get_source_rule_key(desired["hostname"], desired.get("path"))
+                    matched_key, existing = find_container_rule(source_key, "agent", agent_id)
+                    if not existing or not existing.get("rule_ui_override"):
+                        continue
+                    changed, reactivated = restore_rule_lifecycle(existing, desired.get("container_id"))
+                    if existing.get("source_rule_key") is None and matched_key == observed_key:
+                        existing["source_rule_key"] = source_key
+                        existing["lifecycle_generation"] = int(existing.get("lifecycle_generation") or 0) + 1
+                        changed = True
+                    if reactivated:
+                        mark_rule_tunnel_sync_pending(existing)
+                        sync_needed = True
+                    restored_any |= changed
+                    overridden_keys.add(observed_key)
+
+            unresolved_desired = {key: value for key, value in desired_configs.items() if key not in overridden_keys}
+            zone_inventory = get_account_zone_inventory() if unresolved_desired else None
             resolved_desired = {}
-            for rule_key, desired in desired_configs.items():
+            for rule_key, desired in unresolved_desired.items():
                 try:
                     resolved_desired[rule_key] = resolve_account_zone(
                         desired["hostname"],
@@ -245,16 +276,51 @@ def reconcile_agent_report(agent_id, reported_containers):
             with state_lock:
                 for rule_key, selected_zone in resolved_desired.items():
                     desired = desired_configs[rule_key]
-                    existing = managed_rules.get(rule_key)
+                    source_key = get_source_rule_key(desired["hostname"], desired.get("path"))
+                    existing, previous_source_render = restore_container_rule_key(
+                        source_key,
+                        desired["hostname"],
+                        desired.get("path"),
+                        "agent",
+                        agent_id,
+                    )
+                    if previous_source_render:
+                        restored_any = True
+                        sync_needed = True
+                        mark_rule_tunnel_sync_pending(existing)
+                        old_tuple = (
+                            previous_source_render.get("hostname"),
+                            previous_source_render.get("zone_id"),
+                            previous_source_render.get("tunnel_id"),
+                        )
+                        new_tuple = (
+                            existing.get("hostname"),
+                            existing.get("zone_id"),
+                            existing.get("tunnel_id"),
+                        )
+                        if all(old_tuple) and all(new_tuple) and old_tuple != new_tuple:
+                            dns_rekeys.append((old_tuple, new_tuple))
+                    if existing is None:
+                        existing = managed_rules.get(rule_key)
                     target_zone_id = selected_zone["id"]
 
                     if existing:
+                        original_existing = copy.deepcopy(existing)
                         if existing.get("source") == "manual":
                             continue
+                        if existing.get("source") != "agent" or existing.get("agent_id") != agent_id:
+                            logging.warning("[Reconcile-Agent] Ownership mismatch for %s; observation ignored.", rule_key)
+                            continue
+                        if existing.get("source_rule_key") is None:
+                            existing["source_rule_key"] = get_source_rule_key(desired["hostname"], desired.get("path"))
+                            existing["lifecycle_generation"] = int(existing.get("lifecycle_generation") or 0) + 1
+                            restored_any = True
                         if existing.get("rule_ui_override"):
                             if existing.get("status") == "pending_deletion":
                                 existing["status"] = "active"
                                 existing["delete_at"] = None
+                                mark_rule_tunnel_sync_pending(existing)
+                                sync_needed = True
                                 restored_any = True
                             continue
 
@@ -298,6 +364,17 @@ def reconcile_agent_report(agent_id, reported_containers):
                         if changed:
                             restored_any = True
                             logging.info(f"[Reconcile-Agent] Updated rule {rule_key} from agent {agent_id} report.")
+                        tunnel_fields = (
+                            "hostname", "path", "service", "zone_id", "no_tls_verify",
+                            "origin_server_name", "http_host_header", "http2_origin",
+                            "disable_chunked_encoding", "match_sni_to_host", "tunnel_id",
+                        )
+                        if (
+                            original_existing.get("status") != existing.get("status")
+                            or any(original_existing.get(field) != existing.get(field) for field in tunnel_fields)
+                        ):
+                            mark_rule_tunnel_sync_pending(existing)
+                            sync_needed = True
                     else:
                         managed_rules[rule_key] = {
                             "hostname": desired["hostname"],
@@ -321,9 +398,15 @@ def reconcile_agent_report(agent_id, reported_containers):
                             "agent_id": agent_id,
                             "access_group_id": None,
                             "tunnel_name": agent_record.get("assigned_tunnel_name"),
-                            "tunnel_id": agent_record.get("assigned_tunnel_id")
+                            "tunnel_id": agent_record.get("assigned_tunnel_id"),
+                            "source_rule_key": get_source_rule_key(desired["hostname"], desired.get("path")),
+                            "tunnel_sync_pending": True,
+                            "tunnel_sync_last_attempt_at": None,
+                            "tunnel_sync_attempts": 0,
+                            "lifecycle_generation": 0,
                         }
                         restored_any = True
+                        sync_needed = True
                         logging.info(f"[Reconcile-Agent] Created missing rule {rule_key} for agent {agent_id} based on agent report.")
 
                 if restored_any:
@@ -334,11 +417,40 @@ def reconcile_agent_report(agent_id, reported_containers):
                     save_state()
                     publish_state_event('snapshot_refresh')
 
-            if restored_any:
+            if sync_needed:
                 try:
-                    logging.info(f"[Reconcile-Agent] Triggering Cloudflare tunnel config update after restoring rules for agent {agent_id}.")
+                    logging.info(f"[Reconcile-Agent] Triggering pending tunnel synchronization for agent {agent_id}.")
+                    def sync_agent_changes():
+                        retry_pending_tunnel_sync(force=True)
+                        for old_tuple, new_tuple in dns_rekeys:
+                            with state_lock:
+                                new_rule_ready = any(
+                                    rule.get("status") == "active"
+                                    and not rule.get("tunnel_sync_pending")
+                                    and (
+                                        rule.get("hostname"), rule.get("zone_id"),
+                                        _effective_rule_tunnel_id(rule),
+                                    ) == new_tuple
+                                    for rule in managed_rules.values()
+                                )
+                            if not new_rule_ready:
+                                continue
+                            dns_result = create_cloudflare_dns_record(new_tuple[1], new_tuple[0], new_tuple[2])
+                            if not dns_result or dns_result in {"semaphore_timeout", "existing_record_unconfirmed"}:
+                                continue
+                            with state_lock:
+                                old_tuple_owned = any(
+                                    rule.get("status") == "active"
+                                    and (
+                                        rule.get("hostname"), rule.get("zone_id"),
+                                        _effective_rule_tunnel_id(rule),
+                                    ) == old_tuple
+                                    for rule in managed_rules.values()
+                                )
+                            if not old_tuple_owned:
+                                delete_cloudflare_dns_record(old_tuple[1], old_tuple[0], old_tuple[2])
 
-                    t = threading.Thread(target=update_cloudflare_config, args=(agent_record.get("assigned_tunnel_id"),), name=f"agent-reconcile-{agent_id}", daemon=True)
+                    t = threading.Thread(target=sync_agent_changes, name=f"agent-reconcile-{agent_id}", daemon=True)
                     t.start()
                 except Exception as e:
                     logging.error(f"[Reconcile-Agent] Failed to trigger Cloudflare update: {e}", exc_info=True)
@@ -410,10 +522,35 @@ def _run_reconciliation_logic():
         current_app.reconciliation_info["processed_items"] = 0 
         processed_reconcile_items = 0
         hostnames_requiring_dns_setup = set()
+        dns_cleanup_after_sync = set()
         policy_jobs_map = {}
-        zone_inventory = get_account_zone_inventory()
+        overridden_observed_keys = set()
+        effective_tunnel_id = tunnel_state.get("id") if not config.USE_EXTERNAL_CLOUDFLARED else config.EXTERNAL_TUNNEL_ID
+        with state_lock:
+            for observed_key, desired_details in running_labeled_rules_details.items():
+                source_key = get_source_rule_key(desired_details["hostname"], desired_details.get("path"))
+                matched_key, existing_rule = find_container_rule(source_key, "docker")
+                if not existing_rule or not existing_rule.get("rule_ui_override"):
+                    continue
+                changed, reactivated = restore_rule_lifecycle(existing_rule, desired_details["container_id"])
+                if existing_rule.get("source_rule_key") is None and matched_key == observed_key:
+                    existing_rule["source_rule_key"] = source_key
+                    existing_rule["lifecycle_generation"] = int(existing_rule.get("lifecycle_generation") or 0) + 1
+                    changed = True
+                if reactivated:
+                    mark_rule_tunnel_sync_pending(existing_rule)
+                    needs_tunnel_config_update = True
+                state_changed_locally |= changed
+                overridden_observed_keys.add(observed_key)
+                if existing_rule.get("hostname") and existing_rule.get("zone_id"):
+                    hostnames_requiring_dns_setup.add((existing_rule["hostname"], existing_rule["zone_id"], existing_rule.get("tunnel_id") or effective_tunnel_id))
         resolved_running_rules = {}
-        for rule_key, desired_details in running_labeled_rules_details.items():
+        rules_requiring_resolution = {
+            key: details for key, details in running_labeled_rules_details.items()
+            if key not in overridden_observed_keys
+        }
+        zone_inventory = get_account_zone_inventory() if rules_requiring_resolution else None
+        for rule_key, desired_details in rules_requiring_resolution.items():
             try:
                 resolved_running_rules[rule_key] = resolve_account_zone(
                     desired_details["hostname"],
@@ -428,7 +565,6 @@ def _run_reconciliation_logic():
         with state_lock:
             now_utc = datetime.now(timezone.utc)
             current_managed_rule_keys_in_state = set(managed_rules.keys())
-            effective_tunnel_id = tunnel_state.get("id") if not config.USE_EXTERNAL_CLOUDFLARED else config.EXTERNAL_TUNNEL_ID
             master_tunnel_name = tunnel_state.get("name")
                             
             for rule_key, selected_zone in resolved_running_rules.items():
@@ -441,13 +577,47 @@ def _run_reconciliation_logic():
                 if time.time() - reconciliation_start_time > max_total_time - 30:
                     break
 
-                existing_rule = managed_rules.get(rule_key)
+                source_key = get_source_rule_key(
+                    desired_details["hostname"],
+                    desired_details.get("path"),
+                )
+                existing_rule, previous_source_render = restore_container_rule_key(
+                    source_key,
+                    desired_details["hostname"],
+                    desired_details.get("path"),
+                    "docker",
+                )
+                if previous_source_render:
+                    state_changed_locally = True
+                    needs_tunnel_config_update = True
+                    mark_rule_tunnel_sync_pending(existing_rule)
+                    if (
+                        previous_source_render.get("hostname")
+                        and previous_source_render.get("zone_id")
+                        and previous_source_render.get("tunnel_id")
+                    ):
+                        dns_cleanup_after_sync.add((
+                            previous_source_render["hostname"],
+                            previous_source_render["zone_id"],
+                            previous_source_render["tunnel_id"],
+                        ))
+                if existing_rule is None:
+                    existing_rule = managed_rules.get(rule_key)
                 if existing_rule and existing_rule.get("source") == "manual":
                     continue
+                if existing_rule and existing_rule.get("source", "docker") != "docker":
+                    logging.warning("[Reconcile] Rule %s belongs to another source; observation ignored.", rule_key)
+                    continue
+                if existing_rule and existing_rule.get("source_rule_key") is None:
+                    existing_rule["source_rule_key"] = get_source_rule_key(desired_details["hostname"], desired_details.get("path"))
+                    existing_rule["lifecycle_generation"] = int(existing_rule.get("lifecycle_generation") or 0) + 1
+                    state_changed_locally = True
                 if existing_rule and existing_rule.get("rule_ui_override"):
                     if existing_rule.get("status") == "pending_deletion":
                         existing_rule["status"] = "active"
                         existing_rule["delete_at"] = None
+                        mark_rule_tunnel_sync_pending(existing_rule)
+                        needs_tunnel_config_update = True
                         state_changed_locally = True
                     if existing_rule.get("hostname") and existing_rule.get("zone_id"):
                         hostnames_requiring_dns_setup.add((existing_rule.get("hostname"), existing_rule.get("zone_id"), existing_rule.get("tunnel_id") or effective_tunnel_id))
@@ -471,12 +641,18 @@ def _run_reconciliation_logic():
                         "access_policy_ui_override": False, "rule_ui_override": False, "source": "docker",
                         "access_group_id": None,
                         "tunnel_name": master_tunnel_name,
-                        "tunnel_id": effective_tunnel_id
+                        "tunnel_id": effective_tunnel_id,
+                        "source_rule_key": get_source_rule_key(desired_details["hostname"], desired_details.get("path")),
+                        "tunnel_sync_pending": True,
+                        "tunnel_sync_last_attempt_at": None,
+                        "tunnel_sync_attempts": 0,
+                        "lifecycle_generation": 0,
                     }
                     existing_rule = managed_rules[rule_key]
                     state_changed_locally = True
                     needs_tunnel_config_update = True
                 else:
+                    original_existing_rule = copy.deepcopy(existing_rule)
                     changed_in_reconcile = False
                     if existing_rule.get("status") == "pending_deletion":
                         existing_rule["status"] = "active"
@@ -528,13 +704,34 @@ def _run_reconciliation_logic():
                     existing_rule["source"] = "docker" 
                     if changed_in_reconcile:
                         state_changed_locally = True
+                    tunnel_fields = (
+                        "hostname", "path", "service", "zone_id", "no_tls_verify",
+                        "origin_server_name", "http_host_header", "http2_origin",
+                        "disable_chunked_encoding", "match_sni_to_host", "tunnel_id",
+                    )
+                    if (
+                        original_existing_rule.get("status") != existing_rule.get("status")
+                        or any(original_existing_rule.get(field) != existing_rule.get(field) for field in tunnel_fields)
+                    ):
+                        mark_rule_tunnel_sync_pending(existing_rule)
                 
                 hostnames_requiring_dns_setup.add((desired_details["hostname"], target_zone_id, effective_tunnel_id))
                 
                 if not existing_rule.get("access_policy_ui_override", False):
                     policy_jobs_map[rule_key] = copy.deepcopy(desired_details)
             
-            rule_keys_in_state_but_not_running = list(current_managed_rule_keys_in_state - set(running_labeled_rules_details.keys()))
+            observed_source_keys = {
+                get_source_rule_key(details["hostname"], details.get("path"))
+                for details in running_labeled_rules_details.values()
+            }
+            rule_keys_in_state_but_not_running = [
+                key for key in current_managed_rule_keys_in_state
+                if key not in running_labeled_rules_details
+                and not (
+                    managed_rules.get(key, {}).get("source_rule_key")
+                    and managed_rules[key]["source_rule_key"] in observed_source_keys
+                )
+            ]
             for rule_key_to_check in rule_keys_in_state_but_not_running:
                 processed_reconcile_items +=1 
                 current_app.reconciliation_info["processed_items"] = processed_reconcile_items
@@ -581,16 +778,28 @@ def _run_reconciliation_logic():
 
         if needs_tunnel_config_update:
             current_app.reconciliation_info["status"] = "Updating Cloudflare tunnel configuration..."
-            if not config.USE_EXTERNAL_CLOUDFLARED:
-                if not update_cloudflare_config():
-                    logging.error("[Reconcile] Failed to update Cloudflare tunnel configuration.")
-                    current_app.reconciliation_info["status"] = "Error: Failed tunnel config update."
-                else:
-                    logging.info("[Reconcile] Cloudflare tunnel configuration updated successfully.")
-                    current_app.reconciliation_info["status"] = "Tunnel configuration updated."
+            if not update_cloudflare_config(effective_tunnel_id):
+                logging.error("[Reconcile] Failed to update Cloudflare tunnel configuration.")
+                current_app.reconciliation_info["status"] = "Error: Failed tunnel config update."
+                notification_manager.emit(
+                    "cloudflare.tunnel_failure",
+                    str(effective_tunnel_id or "master"),
+                    {"tunnel_id": str(effective_tunnel_id or "")[:12], "operation": "reconciliation update"},
+                )
             else:
-                logging.info("[Reconcile] External mode: Skipping DockFlare-managed tunnel config update.")
-                current_app.reconciliation_info["status"] = "Tunnel config update skipped (external mode)."
+                logging.info("[Reconcile] Cloudflare tunnel configuration updated successfully.")
+                current_app.reconciliation_info["status"] = "Tunnel configuration updated."
+                sync_state_changed = False
+                with state_lock:
+                    for rule in managed_rules.values():
+                        if rule.get("source", "docker") == "docker" and _effective_rule_tunnel_id(rule) == effective_tunnel_id and rule.get("tunnel_sync_pending"):
+                            rule["tunnel_sync_pending"] = False
+                            rule["tunnel_sync_last_attempt_at"] = None
+                            rule["tunnel_sync_attempts"] = 0
+                            rule["lifecycle_generation"] = int(rule.get("lifecycle_generation") or 0) + 1
+                            sync_state_changed = True
+                if sync_state_changed:
+                    save_state()
         
         if hostnames_requiring_dns_setup:
             unique_dns_setups = list(hostnames_requiring_dns_setup)
@@ -606,11 +815,48 @@ def _run_reconciliation_logic():
                     break
                 
                 if tunnel_id_dns and not hostname_dns.startswith('*.'):
-                    create_cloudflare_dns_record(zone_id_dns, hostname_dns, tunnel_id_dns)
+                    try:
+                        dns_result = create_cloudflare_dns_record(zone_id_dns, hostname_dns, tunnel_id_dns)
+                    except Exception:
+                        logging.exception("[Reconcile] DNS creation raised for %s.", hostname_dns)
+                        dns_result = None
+                    if not dns_result or dns_result in {"semaphore_timeout", "existing_record_unconfirmed"}:
+                        notification_manager.emit(
+                            "cloudflare.dns_failure",
+                            f"{zone_id_dns}:{hostname_dns}",
+                            {"hostname": hostname_dns, "operation": "reconciliation create"},
+                        )
                     if config.USE_EXTERNAL_CLOUDFLARED:
                         time.sleep(0.1)
                 elif not tunnel_id_dns:
                     logging.error(f"[Reconcile] Cannot setup DNS for {hostname_dns}: Tunnel ID is missing.")
+                    notification_manager.emit(
+                        "cloudflare.tunnel_failure",
+                        hostname_dns,
+                        {"hostname": hostname_dns, "operation": "reconciliation resolve tunnel"},
+                    )
+
+        for old_hostname, old_zone_id, old_tunnel_id in dns_cleanup_after_sync:
+            with state_lock:
+                sync_pending = any(
+                    rule.get("tunnel_sync_pending")
+                    and _effective_rule_tunnel_id(rule) == old_tunnel_id
+                    for rule in managed_rules.values()
+                )
+                still_owned = any(
+                    rule.get("status") == "active"
+                    and rule.get("hostname") == old_hostname
+                    and rule.get("zone_id") == old_zone_id
+                    and _effective_rule_tunnel_id(rule) == old_tunnel_id
+                    for rule in managed_rules.values()
+                )
+            if not sync_pending and not still_owned and not old_hostname.startswith("*."):
+                if not delete_cloudflare_dns_record(old_zone_id, old_hostname, old_tunnel_id):
+                    notification_manager.emit(
+                        "cloudflare.dns_failure",
+                        f"{old_zone_id}:{old_hostname}",
+                        {"hostname": old_hostname, "operation": "reconciliation delete"},
+                    )
                 
         current_app.reconciliation_info["in_progress"] = False
         current_app.reconciliation_info["progress"] = 100 
@@ -648,107 +894,206 @@ def reconcile_state_threaded():
     reconcile_thread.start()
     logging.info(f"Started reconciliation in background thread {reconcile_thread.name}")
 
-def cleanup_expired_rules(stop_event_param):
-    from app import app as main_app_instance_for_context
-    logging.info("Starting cleanup task for expired rules...")
-    if stop_event_param is None:
-        logging.error("cleanup_expired_rules called with None stop_event_param. Task will not run correctly.")
-        return
+def _effective_rule_tunnel_id(rule):
+    explicit_tunnel_id = rule.get("tunnel_id")
+    if explicit_tunnel_id:
+        return explicit_tunnel_id
+    return config.EXTERNAL_TUNNEL_ID if config.USE_EXTERNAL_CLOUDFLARED else tunnel_state.get("id")
 
-    while not stop_event_param.is_set():
-        next_check_time = time.time() + config.CLEANUP_INTERVAL_SECONDS
-        with main_app_instance_for_context.app_context():
-            try:
-                logging.debug("Running cleanup check for expired rules...")
-                rules_to_delete_keys = []
-                now_utc = datetime.now(timezone.utc)
-                state_changed_in_cleanup = False
 
+def _candidate_still_matches(candidate):
+    rule = managed_rules.get(candidate["key"])
+    if not rule:
+        return False
+    return (
+        rule.get("status") == candidate["status"]
+        and rule.get("delete_at") == candidate["delete_at"]
+        and rule.get("container_id") == candidate["container_id"]
+        and rule.get("source", "docker") == candidate["source"]
+        and rule.get("agent_id") == candidate["agent_id"]
+        and rule.get("lifecycle_generation", 0) == candidate["lifecycle_generation"]
+        and rule.get("tunnel_id") == candidate["tunnel_id"]
+    )
+
+
+def cleanup_expired_rules_once(now=None):
+    current_time = now or datetime.now(timezone.utc)
+    candidates = []
+    with state_lock:
+        for key, rule in managed_rules.items():
+            if rule.get("source", "docker") not in {"docker", "agent"} or rule.get("status") != "pending_deletion":
+                continue
+            deadline = rule.get("delete_at")
+            if isinstance(deadline, datetime):
+                deadline = deadline.astimezone(timezone.utc) if deadline.tzinfo else deadline.replace(tzinfo=timezone.utc)
+                if deadline > current_time:
+                    continue
+            candidates.append({
+                "key": key,
+                "status": rule.get("status"),
+                "delete_at": rule.get("delete_at"),
+                "container_id": rule.get("container_id"),
+                "source": rule.get("source", "docker"),
+                "agent_id": rule.get("agent_id"),
+                "lifecycle_generation": rule.get("lifecycle_generation", 0),
+                "tunnel_id": rule.get("tunnel_id"),
+                "effective_tunnel_id": _effective_rule_tunnel_id(rule),
+                "hostname": rule.get("hostname"),
+                "zone_id": rule.get("zone_id"),
+                "access_app_id": rule.get("access_app_id"),
+            })
+
+    groups = {}
+    for candidate in candidates:
+        groups.setdefault(candidate["effective_tunnel_id"], []).append(candidate)
+    committed = 0
+    committed_candidates = []
+    tunnel_failures = []
+    dns_failures = []
+    access_failures = []
+    for tunnel_id in sorted(groups, key=lambda value: str(value or "")):
+        if not tunnel_id:
+            logging.error("Cleanup retained %s rules because their effective tunnel is missing.", len(groups[tunnel_id]))
+            continue
+        with get_tunnel_operation_lock(tunnel_id):
+            with state_lock:
+                valid = [candidate for candidate in groups[tunnel_id] if _candidate_still_matches(candidate)]
+            if not valid:
+                continue
+            if not update_cloudflare_config(tunnel_id):
+                tunnel_failures.append(tunnel_id)
+                continue
+            successful = []
+            for candidate in valid:
                 with state_lock:
-                    for rule_key, details in managed_rules.items():
-                        if details.get("status") == "pending_deletion" and details.get("source", "docker") in ["docker", "agent"]:
-                            delete_at = details.get("delete_at")
-                            if isinstance(delete_at, datetime):
-                                delete_at_utc = delete_at.astimezone(timezone.utc) if delete_at.tzinfo else delete_at.replace(tzinfo=timezone.utc)
-                                if delete_at_utc <= now_utc:
-                                    rules_to_delete_keys.append(rule_key)
-                            else:
-                                logging.warning(f"Rule {rule_key} pending delete but has invalid/missing delete_at. Marking for immediate deletion.")
-                                rules_to_delete_keys.append(rule_key)
-
-                    for rule_key, details in managed_rules.items():
-                        if details.get("source") == "manual" and details.get("status") == "pending_deletion":
-                            logging.warning(f"{details.get('source', 'unknown').title()} rule {rule_key} found 'pending_deletion'. Resetting to 'active'.")
-                            details["status"] = "active"
-                            details["delete_at"] = None
-                            state_changed_in_cleanup = True
-                
-                if state_changed_in_cleanup and not rules_to_delete_keys:
+                    if not _candidate_still_matches(candidate):
+                        continue
+                    dns_shared = any(
+                        key != candidate["key"]
+                        and rule.get("hostname") == candidate["hostname"]
+                        and rule.get("zone_id") == candidate["zone_id"]
+                        and _effective_rule_tunnel_id(rule) == tunnel_id
+                        for key, rule in managed_rules.items()
+                    )
+                    app_shared = candidate["access_app_id"] and any(
+                        key != candidate["key"] and rule.get("access_app_id") == candidate["access_app_id"]
+                        for key, rule in managed_rules.items()
+                    )
+                if candidate["hostname"] and not dns_shared and not delete_cloudflare_dns_record(candidate["zone_id"], candidate["hostname"], tunnel_id):
+                    dns_failures.append(candidate)
+                    continue
+                if candidate["access_app_id"] and not app_shared and not delete_cloudflare_access_application(candidate["access_app_id"]):
+                    access_failures.append(candidate)
+                    continue
+                successful.append(candidate)
+            with state_lock:
+                for candidate in successful:
+                    if _candidate_still_matches(candidate):
+                        del managed_rules[candidate["key"]]
+                        committed += 1
+                        committed_candidates.append(candidate)
+                if successful:
                     save_state()
+    if committed:
+        publish_state_event("snapshot_refresh")
+        notification_manager.emit(
+            "rule.deleted",
+            f"cleanup:{int(current_time.timestamp())}",
+            {
+                "source": "cleanup",
+                "resources": [
+                    {
+                        "key": candidate["key"],
+                        "hostname": candidate.get("hostname"),
+                        "source": candidate.get("source"),
+                    }
+                    for candidate in committed_candidates
+                ],
+                "public_url": config.DOCKFLARE_PUBLIC_URL,
+            },
+        )
+    for tunnel_id in tunnel_failures:
+        notification_manager.emit(
+            "cloudflare.tunnel_failure",
+            tunnel_id,
+            {"tunnel_id": str(tunnel_id)[:12], "operation": "cleanup update"},
+        )
+    for candidate in dns_failures:
+        notification_manager.emit(
+            "cloudflare.dns_failure",
+            f"{candidate.get('zone_id')}:{candidate.get('hostname')}",
+            {"hostname": candidate.get("hostname"), "operation": "delete", "source": candidate.get("source")},
+        )
+    for candidate in access_failures:
+        notification_manager.emit(
+            "cloudflare.access_failure",
+            candidate["key"],
+            {"hostname": candidate.get("hostname"), "operation": "delete", "source": candidate.get("source")},
+        )
+    return committed
 
-                if rules_to_delete_keys:
-                    logging.info(f"Found {len(rules_to_delete_keys)} expired rules to process: {rules_to_delete_keys}")
-                    effective_tunnel_id = tunnel_state.get("id") if not config.USE_EXTERNAL_CLOUDFLARED else config.EXTERNAL_TUNNEL_ID
 
-                    rules_being_deleted = {}
-                    with state_lock:
-                        for key in rules_to_delete_keys:
-                            if key in managed_rules:
-                                rules_being_deleted[key] = managed_rules[key]
+def retry_pending_tunnel_sync(now=None, force=False):
+    current_time = now or datetime.now(timezone.utc)
+    pending_tunnels = set()
+    with state_lock:
+        for rule in managed_rules.values():
+            if rule.get("tunnel_sync_pending"):
+                tunnel_id = _effective_rule_tunnel_id(rule)
+                if tunnel_id:
+                    pending_tunnels.add(tunnel_id)
+    changed = False
+    for tunnel_id in sorted(pending_tunnels):
+        with state_lock:
+            rules = [rule for rule in managed_rules.values() if rule.get("tunnel_sync_pending") and _effective_rule_tunnel_id(rule) == tunnel_id]
+            eligible = force
+            if not eligible:
+                for rule in rules:
+                    last_attempt = rule.get("tunnel_sync_last_attempt_at")
+                    if not isinstance(last_attempt, datetime) or (current_time - last_attempt).total_seconds() >= config.TUNNEL_SYNC_RETRY_SECONDS:
+                        eligible = True
+                        break
+            if not eligible:
+                continue
+            for rule in rules:
+                rule["tunnel_sync_last_attempt_at"] = current_time
+                rule["tunnel_sync_attempts"] = int(rule.get("tunnel_sync_attempts") or 0) + 1
+                changed = True
+        success = update_cloudflare_config(tunnel_id)
+        if not success:
+            notification_manager.emit(
+                "cloudflare.tunnel_failure",
+                tunnel_id,
+                {
+                    "tunnel_id": str(tunnel_id)[:12],
+                    "operation": "retry pending synchronization",
+                    "retry_count": max((int(rule.get("tunnel_sync_attempts") or 0) for rule in rules), default=0),
+                },
+            )
+        if success:
+            with state_lock:
+                for rule in managed_rules.values():
+                    if rule.get("tunnel_sync_pending") and _effective_rule_tunnel_id(rule) == tunnel_id:
+                        rule["tunnel_sync_pending"] = False
+                        rule["tunnel_sync_last_attempt_at"] = None
+                        rule["tunnel_sync_attempts"] = 0
+                        rule["lifecycle_generation"] = int(rule.get("lifecycle_generation") or 0) + 1
+                        changed = True
+    if changed:
+        save_state()
+        publish_state_event("snapshot_refresh")
+    return changed
 
-                    for rule_key, delete_info in rules_being_deleted.items():
-                        hostname_del = delete_info.get("hostname")
-                        access_app_id_del = delete_info.get("access_app_id")
 
-                        is_hostname_still_used = any(
-                            k not in rules_to_delete_keys and r.get("hostname") == hostname_del
-                            for k, r in managed_rules.items()
-                        )
-
-                        if hostname_del and not is_hostname_still_used:
-                            logging.info(f"Hostname '{hostname_del}' is no longer used. Deleting its DNS record.")
-                            delete_cloudflare_dns_record(delete_info.get("zone_id"), hostname_del, effective_tunnel_id)
-                        elif hostname_del:
-                            logging.info(f"Skipping DNS delete for '{hostname_del}' as it is still used by other rules.")
-
-                        is_app_id_still_used = any(
-                            k not in rules_to_delete_keys and r.get("access_app_id") == access_app_id_del
-                            for k, r in managed_rules.items()
-                        )
-
-                        if access_app_id_del and not is_app_id_still_used:
-                            logging.info(f"Access App ID '{access_app_id_del}' is no longer used. Deleting it.")
-                            delete_cloudflare_access_application(access_app_id_del)
-                        elif access_app_id_del:
-                            logging.info(f"Skipping Access App delete for '{access_app_id_del}' as it is still used by other rules.")
-
-                    config_updated = False
-                    if not config.USE_EXTERNAL_CLOUDFLARED:
-                        logging.info("Updating Cloudflare config to remove expired ingress rules...")
-                        if update_cloudflare_config():
-                            config_updated = True
-                        else:
-                            logging.error("Failed to update Cloudflare tunnel config during rule cleanup. Rules will remain in local state temporarily.")
-                    else:
-                        config_updated = True
-
-                    if config_updated:
-                        with state_lock:
-                            deleted_count = 0
-                            for key in rules_to_delete_keys:
-                                if key in managed_rules:
-                                    del managed_rules[key]
-                                    deleted_count += 1
-                            if deleted_count > 0:
-                                logging.info(f"Removed {deleted_count} expired rules from local state.")
-                                save_state()
-                                publish_state_event('snapshot_refresh')
-
-            except Exception as e_cleanup:
-                logging.error(f"Error in cleanup task loop: {e_cleanup}", exc_info=True)
-
-        wait_duration = max(0, next_check_time - time.time())
-        if not stop_event_param.is_set():
-            stop_event_param.wait(wait_duration)
-
-    logging.info("Cleanup task for expired rules stopped.")
+def cleanup_expired_rules(stop_event_param):
+    from app import app as main_app
+    if stop_event_param is None:
+        return
+    while not stop_event_param.is_set():
+        with main_app.app_context():
+            try:
+                cleanup_expired_rules_once()
+                retry_pending_tunnel_sync()
+            except Exception:
+                logging.exception("Expired-rule cleanup pass failed.")
+        stop_event_param.wait(config.CLEANUP_INTERVAL_SECONDS)

dockflare/app/core/reusable_policies.py

@@ -432,7 +432,7 @@ def delete_access_group_and_policy(group_id):
         logging.warning(f"Access group '{group_id}' not found in state")
         return False
 
-    group = access_groups[group_id]
+    group = copy.deepcopy(access_groups[group_id])
     policy_id = group.get("cloudflare_policy_id")
 
     if policy_id:
@@ -444,6 +444,9 @@ def delete_access_group_and_policy(group_id):
         logging.info(f"No Cloudflare policy ID found for access group '{group_id}', removing from state only")
 
     del access_groups[group_id]
-    save_state()
+    if not save_state():
+        access_groups[group_id] = group
+        logging.error(f"Failed to persist removal of access group '{group_id}'")
+        return False
     logging.info(f"Removed access group '{group_id}' from state")
     return True

dockflare/app/core/service_token_manager.py

@@ -174,6 +174,7 @@ def generate_compose_content(key_id, public_url, cloudflared_image="cloudflare/c
       - PING=1
       - INFO=1
       - EXEC=1
+      - DELETE=0
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     networks:

dockflare/app/core/state_manager.py

@@ -24,13 +24,32 @@ from typing import Dict, Any, List
 from app import config
 from app.core import agent_key_store
 from app.core.access_policy_rules import normalize_managed_access_group
-from app.core.utils import get_rule_key
+from app.core.utils import get_label, get_rule_key, get_source_rule_key
+
+STATE_SCHEMA_VERSION = 3
+RULE_LIFECYCLE_DEFAULTS = {
+    "source_rule_key": None,
+    "tunnel_sync_pending": False,
+    "tunnel_sync_last_attempt_at": None,
+    "tunnel_sync_attempts": 0,
+    "lifecycle_generation": 0,
+}
+AGENT_PROTOCOL_DEFAULTS = {
+    "protocol_version": None,
+    "agent_session_id": None,
+    "last_event_sequence": 0,
+    "last_report_sequence": 0,
+    "last_complete_containers": None,
+}
 
 managed_rules = {}
 access_groups = {}
 agents = {}
+agent_decommissions = {}
 identity_providers = {}
 agent_cf_token = {}
+state_extensions = {}
+_state_write_blocked_reason = None
 state_lock = threading.RLock()
 logging.info(
     "STATE_MANAGER_INIT: managed_rules ID: %s, access_groups ID: %s, agents ID: %s, identity_providers ID: %s",
@@ -40,6 +59,117 @@ logging.info(
     id(identity_providers)
 )
 
+
+def restore_rule_lifecycle(rule, container_id):
+    """Apply only runtime-owned lifecycle fields for a valid observation."""
+    changed = False
+    reactivated = False
+    if rule.get("container_id") != container_id:
+        rule["container_id"] = container_id
+        changed = True
+    if rule.get("status") != "active":
+        rule["status"] = "active"
+        rule["delete_at"] = None
+        changed = True
+        reactivated = True
+    elif rule.get("delete_at") is not None:
+        rule["delete_at"] = None
+        changed = True
+        reactivated = True
+    if changed:
+        rule["lifecycle_generation"] = int(rule.get("lifecycle_generation") or 0) + 1
+    return changed, reactivated
+
+
+def find_container_rule(observed_rule_key, source, agent_id=None):
+    """Find one compatible container-backed rule while the caller holds state_lock."""
+    def compatible(rule):
+        if rule.get("source", "docker") != source or source not in {"docker", "agent"}:
+            return False
+        return source != "agent" or rule.get("agent_id") == agent_id
+
+    direct = managed_rules.get(observed_rule_key)
+    if direct and compatible(direct):
+        competing = [
+            key for key, rule in managed_rules.items()
+            if key != observed_rule_key
+            and rule.get("source_rule_key") == observed_rule_key
+            and compatible(rule)
+        ]
+        if competing:
+            raise ValueError(f"Ambiguous container rule identity: {observed_rule_key}")
+        return observed_rule_key, direct
+    matches = [
+        (key, rule) for key, rule in managed_rules.items()
+        if rule.get("source_rule_key") == observed_rule_key and compatible(rule)
+    ]
+    if len(matches) > 1:
+        raise ValueError(f"Ambiguous container rule identity: {observed_rule_key}")
+    return matches[0] if matches else (None, None)
+
+
+def restore_container_rule_key(observed_rule_key, hostname, path, source, agent_id=None):
+    """Move one label-controlled rule back to its immutable source key."""
+    matched_key, rule = find_container_rule(observed_rule_key, source, agent_id)
+    if not rule or matched_key == observed_rule_key or rule.get("rule_ui_override"):
+        return rule, None
+
+    previous = {
+        "key": matched_key,
+        "hostname": rule.get("hostname"),
+        "zone_id": rule.get("zone_id"),
+        "tunnel_id": rule.get("tunnel_id"),
+    }
+    managed_rules.pop(matched_key)
+    managed_rules[observed_rule_key] = rule
+    rule["hostname"] = hostname
+    rule["path"] = path
+    rule["source_rule_key"] = observed_rule_key
+    rule["lifecycle_generation"] = int(rule.get("lifecycle_generation") or 0) + 1
+    return rule, previous
+
+
+def agent_inventory_contains_rule(containers, rule):
+    """Return whether a stored Agent inventory contains the rule's current binding."""
+    if not isinstance(containers, list):
+        return False
+    container_id = rule.get("container_id")
+    source_rule_key = rule.get("source_rule_key")
+    if not container_id or not source_rule_key:
+        return False
+    for container in containers:
+        if not isinstance(container, dict) or container.get("id") != container_id:
+            continue
+        labels = container.get("labels") or {}
+        hostname = get_label(labels, "hostname")
+        if hostname:
+            try:
+                if get_source_rule_key(hostname, get_label(labels, "path")) == source_rule_key:
+                    return True
+            except ValueError:
+                pass
+        index = 0
+        while True:
+            hostname = get_label(labels, f"{index}.hostname")
+            if not hostname:
+                break
+            path = get_label(labels, f"{index}.path", get_label(labels, "path"))
+            try:
+                if get_source_rule_key(hostname, path) == source_rule_key:
+                    return True
+            except ValueError:
+                pass
+            index += 1
+    return False
+
+
+def mark_rule_tunnel_sync_pending(rule):
+    if rule.get("tunnel_sync_pending"):
+        return False
+    rule["tunnel_sync_pending"] = True
+    rule["lifecycle_generation"] = int(rule.get("lifecycle_generation") or 0) + 1
+    return True
+
 def _deserialize_datetime(dt_str):
     if not dt_str:
         return None
@@ -54,14 +184,19 @@ def _deserialize_datetime(dt_str):
         return None
 
 def load_state():
+    global _state_write_blocked_reason
     logging.info(f"LOAD_STATE: Start. Initial managed_rules ID: {id(managed_rules)}, Current len: {len(managed_rules)}")
     state_dir = os.path.dirname(config.STATE_FILE_PATH)
     
     with state_lock:
+        _state_write_blocked_reason = None
         managed_rules.clear()
         access_groups.clear()
+        agents.clear()
+        agent_decommissions.clear()
         identity_providers.clear()
         agent_cf_token.clear()
+        state_extensions.clear()
         logging.info(
             "LOAD_STATE: After .clear(), managed_rules ID: %s, len: %s",
             id(managed_rules),
@@ -85,6 +220,12 @@ def load_state():
             with open(config.STATE_FILE_PATH, 'r') as f:
                 loaded_data = json.load(f)
 
+            schema_version = loaded_data.get("state_schema_version", 1) if isinstance(loaded_data, dict) and "managed_rules" in loaded_data else 1
+            if isinstance(schema_version, bool) or not isinstance(schema_version, int) or schema_version > STATE_SCHEMA_VERSION:
+                _state_write_blocked_reason = f"Unsupported state schema version: {schema_version}"
+                raise ValueError(f"Unsupported state schema version: {schema_version}")
+            _state_write_blocked_reason = None
+
             rules_to_load = {}
             groups_to_load = {}
 
@@ -93,12 +234,16 @@ def load_state():
                 rules_to_load = loaded_data.get("managed_rules", {})
                 groups_to_load = loaded_data.get("access_groups", {})
                 agents_to_load = loaded_data.get("agents", {})
+                decommissions_to_load = loaded_data.get("agent_decommissions", {})
                 idps_to_load = loaded_data.get("identity_providers", {})
                 cf_token_to_load = loaded_data.get("agent_cf_token", {})
+                known_top_level = {"state_schema_version", "managed_rules", "access_groups", "agents", "agent_decommissions", "identity_providers", "agent_cf_token"}
+                state_extensions.update({key: value for key, value in loaded_data.items() if key not in known_top_level})
             else:
                 logging.info("Loading state from old format (rules only). Will migrate on next save.")
                 rules_to_load = loaded_data
                 agents_to_load = {}
+                decommissions_to_load = {}
                 idps_to_load = {}
                 cf_token_to_load = {}
 
@@ -108,7 +253,17 @@ def load_state():
                 access_groups[group_id] = normalized_group
                 if changed:
                     group_migration_count += 1
-            agents.update(agents_to_load)
+            for agent_id, agent_data in agents_to_load.items():
+                agent_copy = dict(agent_data)
+                for field, default in AGENT_PROTOCOL_DEFAULTS.items():
+                    agent_copy.setdefault(field, default)
+                agents[agent_id] = agent_copy
+            if isinstance(decommissions_to_load, dict):
+                agent_decommissions.update(
+                    (operation_id, dict(operation))
+                    for operation_id, operation in decommissions_to_load.items()
+                    if isinstance(operation_id, str) and isinstance(operation, dict)
+                )
             identity_providers.update(idps_to_load)
             agent_cf_token.update(cf_token_to_load)
             key_count = len(agent_key_store.list_keys())
@@ -139,6 +294,11 @@ def load_state():
                     rule_copy["delete_at"] = _deserialize_datetime(delete_at_val)
                 elif not isinstance(delete_at_val, (datetime, type(None))):
                     rule_copy["delete_at"] = None
+                sync_attempt_val = rule_copy.get("tunnel_sync_last_attempt_at")
+                if isinstance(sync_attempt_val, str):
+                    rule_copy["tunnel_sync_last_attempt_at"] = _deserialize_datetime(sync_attempt_val)
+                elif not isinstance(sync_attempt_val, (datetime, type(None))):
+                    rule_copy["tunnel_sync_last_attempt_at"] = None
 
                 rule_copy.setdefault("zone_id", None)
                 rule_copy.setdefault("access_app_id", None)
@@ -159,6 +319,8 @@ def load_state():
                 rule_copy.setdefault("http2_origin", False)
                 rule_copy.setdefault("disable_chunked_encoding", False)
                 rule_copy.setdefault("match_sni_to_host", False)
+                for field, default in RULE_LIFECYCLE_DEFAULTS.items():
+                    rule_copy.setdefault(field, default)
 
                 tunnel_name = rule_copy.get("tunnel_name")
                 if not tunnel_name or tunnel_name == "dockflare-tunnel":
@@ -168,7 +330,7 @@ def load_state():
 
                 managed_rules[final_key] = rule_copy
 
-            migration_needed = migrated_count > 0 or tunnel_name_migration_count > 0 or group_migration_count > 0
+            migration_needed = schema_version < STATE_SCHEMA_VERSION or migrated_count > 0 or tunnel_name_migration_count > 0 or group_migration_count > 0
             if migrated_count > 0:
                 logging.info(f"LOAD_STATE: Migrated {migrated_count} rules to the new key format.")
             if tunnel_name_migration_count > 0:
@@ -179,7 +341,7 @@ def load_state():
                 save_state()
             
             logging.info(f"LOAD_STATE: Loaded {len(managed_rules)} rules. managed_rules ID after populating: {id(managed_rules)}")
-        except (json.JSONDecodeError, IOError, OSError) as e:
+        except (json.JSONDecodeError, IOError, OSError, ValueError) as e:
             logging.error(f"LOAD_STATE: Error loading state from {config.STATE_FILE_PATH}: {e}. Starting fresh (already cleared).", exc_info=True)
         except Exception as e_load_unexp:
             logging.error(f"LOAD_STATE: Unexpected error loading state: {e_load_unexp}. Starting fresh (already cleared).", exc_info=True)
@@ -482,6 +644,12 @@ def ensure_authenticated_default_policy(flask_app=None):
 
 def save_state():
     with state_lock:
+        if _state_write_blocked_reason:
+            logging.error(
+                "SAVE_STATE: Refusing to overwrite state after a failed compatibility check: %s",
+                _state_write_blocked_reason,
+            )
+            return False
         current_thread_name = threading.current_thread().name
         logging.info(f"SAVE_STATE: Start. THREAD: {current_thread_name}. Items to save: {len(managed_rules)} rules, {len(access_groups)} access groups.")
 
@@ -489,6 +657,7 @@ def save_state():
         rules_to_iterate = list(managed_rules.items())
         groups_to_iterate = dict(access_groups)
         agents_to_iterate = dict(agents)
+        decommissions_to_iterate = dict(agent_decommissions)
         idps_to_iterate = dict(identity_providers)
         cf_token_to_iterate = dict(agent_cf_token)
 
@@ -521,18 +690,28 @@ def save_state():
                     "tunnel_id": rule.get("tunnel_id"),
                     "tunnel_name": rule.get("tunnel_name")
                 }
+                for field in RULE_LIFECYCLE_DEFAULTS:
+                    data_to_serialize[field] = rule.get(field, RULE_LIFECYCLE_DEFAULTS[field])
+                for field, value in rule.items():
+                    data_to_serialize.setdefault(field, value)
                 delete_at_val = rule.get("delete_at")
                 if isinstance(delete_at_val, datetime):
                     data_to_serialize["delete_at"] = delete_at_val.astimezone(timezone.utc).isoformat().replace('+00:00', 'Z')
+                sync_attempt_val = rule.get("tunnel_sync_last_attempt_at")
+                if isinstance(sync_attempt_val, datetime):
+                    data_to_serialize["tunnel_sync_last_attempt_at"] = sync_attempt_val.astimezone(timezone.utc).isoformat().replace('+00:00', 'Z')
                 serializable_rules[rule_key] = data_to_serialize
             except Exception as e_serialize_item:
                 logging.error(f"SAVE_STATE_LOOP_ERROR: THREAD: {current_thread_name}. Error preparing rule for serialization '{rule_key}': {e_serialize_item}. Rule data: {rule}", exc_info=True)
                 continue
         
         final_state_to_save = {
+            **state_extensions,
+            "state_schema_version": STATE_SCHEMA_VERSION,
             "managed_rules": serializable_rules,
             "access_groups": groups_to_iterate,
             "agents": agents_to_iterate,
+            "agent_decommissions": decommissions_to_iterate,
             "identity_providers": idps_to_iterate,
             "agent_cf_token": cf_token_to_iterate
         }
@@ -544,16 +723,16 @@ def save_state():
                     os.makedirs(state_dir, exist_ok=True)
                 except OSError as e_mkdir:
                     logging.error(f"SAVE_STATE: THREAD: {current_thread_name}. Mkdir error {e_mkdir}. Save failed.")
-                    return
+                    return False
             temp_file_path = config.STATE_FILE_PATH + ".tmp"
             with open(temp_file_path, 'w') as f:
                 json.dump(final_state_to_save, f, indent=2)
             os.replace(temp_file_path, config.STATE_FILE_PATH)
             logging.info(f"SAVE_STATE: THREAD: {current_thread_name}. Successfully saved state to {config.STATE_FILE_PATH}")
+            return True
         except Exception as e_save_io:
             logging.error(f"SAVE_STATE: THREAD: {current_thread_name}. File I/O or other error: {e_save_io}", exc_info=True)
-        
-        logging.info(f"SAVE_STATE: End. THREAD: {current_thread_name}.")
+            return False
 
 def add_agent(agent_id, agent_data):
     """
@@ -627,8 +806,7 @@ def find_agent_id_by_key(key_id):
     info = agent_key_store.get_key(key_id)
     if not info:
         return None
-    owner = info.get('owner')
-    return owner
+    return info.get('bound_agent_id') or info.get('owner')
 
 def get_agent_key_info(key_id):
     """Return metadata for a given key or None."""
@@ -779,7 +957,9 @@ def serialize_managed_rule(rule_key: str, rule: Dict[str, Any]) -> Dict[str, Any
         "tunnel_name": rule.get("tunnel_name"),
         "access_policy_type": rule.get("access_policy_type"),
         "access_policy_ui_override": rule.get("access_policy_ui_override", False),
-        "rule_ui_override": rule.get("rule_ui_override", False)
+        "rule_ui_override": rule.get("rule_ui_override", False),
+        "source_rule_key": rule.get("source_rule_key"),
+        "tunnel_sync_pending": rule.get("tunnel_sync_pending", False),
     }
 
 

dockflare/app/core/tunnel_manager.py

@@ -17,17 +17,43 @@
 # dockflare/app/core/tunnel_manager.py
 import copy
 import logging
-import json 
 import time 
+import threading
+from datetime import datetime, timezone
 from flask import current_app
 from app import config, docker_client
 from app import tunnel_state, cloudflared_agent_state 
 from app.core import cloudflare_api
-from app.core.state_manager import managed_rules, state_lock, list_agents, get_agent_rules 
+from app.core.state_manager import managed_rules, state_lock
 
 from docker.errors import NotFound, APIError 
 import requests 
 
+_tunnel_operation_locks = {}
+_tunnel_operation_locks_guard = threading.Lock()
+
+
+def get_tunnel_operation_lock(tunnel_id):
+    normalized_id = str(tunnel_id or "").strip()
+    if not normalized_id:
+        raise ValueError("tunnel_id is required")
+    with _tunnel_operation_locks_guard:
+        return _tunnel_operation_locks.setdefault(normalized_id, threading.RLock())
+
+
+def is_rule_ingress_eligible(rule, now=None):
+    if rule.get("status") == "active":
+        return True
+    if rule.get("status") != "pending_deletion":
+        return False
+    deadline = rule.get("delete_at")
+    if not isinstance(deadline, datetime):
+        return False
+    deadline = deadline.astimezone(timezone.utc) if deadline.tzinfo else deadline.replace(tzinfo=timezone.utc)
+    current = now or datetime.now(timezone.utc)
+    current = current.astimezone(timezone.utc) if current.tzinfo else current.replace(tzinfo=timezone.utc)
+    return deadline > current
+
 def initialize_tunnel():
     from app import app
     with app.app_context():
@@ -128,93 +154,83 @@ def update_cloudflare_config(target_tunnel_id=None):
             return False
 
         logging.info(f"Preparing Cloudflare config update for tunnel {target_tunnel_id}")
+        operation_lock = get_tunnel_operation_lock(target_tunnel_id)
+        with operation_lock:
+            return _update_cloudflare_config_locked(target_tunnel_id, master_tunnel_id)
 
-        desired_ingress = _build_ingress_entries_for_tunnel(target_tunnel_id, master_tunnel_id)
 
-        try:
-            current_api_config_ruleset = cloudflare_api.get_current_cf_config(target_tunnel_id)
-        except Exception as exc:
-            logging.error(f"Failed to fetch current CF config for tunnel {target_tunnel_id}: {exc}")
-            if target_tunnel_id == master_tunnel_id:
-                tunnel_state["error"] = f"Failed get tunnel config: {exc}"
-            return False
+def _update_cloudflare_config_locked(target_tunnel_id, master_tunnel_id):
+    desired_ingress = _build_ingress_entries_for_tunnel(target_tunnel_id, master_tunnel_id)
 
-        if current_api_config_ruleset is None:
-            logging.error(f"Cloudflare returned no config for tunnel {target_tunnel_id}.")
-            return False
+    try:
+        current_api_config_ruleset = cloudflare_api.get_current_cf_config(target_tunnel_id)
+    except Exception as exc:
+        logging.error(f"Failed to fetch current CF config for tunnel {target_tunnel_id}: {exc}")
+        if target_tunnel_id == master_tunnel_id:
+            tunnel_state["error"] = f"Failed get tunnel config: {exc}"
+        return False
 
-        current_api_ingress_rules = current_api_config_ruleset.get("ingress", []) or []
-        desired_comparable = {_ingress_to_comparable(rule) for rule in desired_ingress}
-        preserved_rules = []
+    if current_api_config_ruleset is None:
+        logging.error(f"Cloudflare returned no config for tunnel {target_tunnel_id}.")
+        return False
 
-        for api_rule in current_api_ingress_rules:
-            if _is_catch_all_rule(api_rule):
+    current_api_ingress_rules = current_api_config_ruleset.get("ingress", []) or []
+    desired_comparable = {_ingress_to_comparable(rule) for rule in desired_ingress}
+    preserved_rules = []
+    for api_rule in current_api_ingress_rules:
+        if _is_catch_all_rule(api_rule):
+            preserved_rules.append(api_rule)
+            continue
+        hostname = api_rule.get("hostname") or ""
+        if "*" in hostname:
+            comp = _ingress_to_comparable(api_rule)
+            if comp not in desired_comparable:
                 preserved_rules.append(api_rule)
-                continue
-            hostname = api_rule.get("hostname") or ""
-            if "*" in hostname:
-                comp = _ingress_to_comparable(api_rule)
-                if comp not in desired_comparable:
-                    preserved_rules.append(api_rule)
 
-        if config.PRESERVE_UNMANAGED_CF_INGRESS_FIELDS:
-            final_ingress = [_merge_desired_with_current_fields(rule, current_api_ingress_rules) for rule in desired_ingress]
-        else:
-            final_ingress = list(desired_ingress)
-        seen = set(desired_comparable)
-        for preserved in preserved_rules:
-            comp = _ingress_to_comparable(preserved)
-            if comp not in seen:
-                final_ingress.append(preserved)
-                seen.add(comp)
+    if config.PRESERVE_UNMANAGED_CF_INGRESS_FIELDS:
+        final_ingress = [_merge_desired_with_current_fields(rule, current_api_ingress_rules) for rule in desired_ingress]
+    else:
+        final_ingress = list(desired_ingress)
+    seen = set(desired_comparable)
+    for preserved in preserved_rules:
+        comp = _ingress_to_comparable(preserved)
+        if comp not in seen:
+            final_ingress.append(preserved)
+            seen.add(comp)
+    if not any(_is_catch_all_rule(rule) for rule in final_ingress):
+        final_ingress.append({"service": "http_status:404"})
 
-        if not any(_is_catch_all_rule(rule) for rule in final_ingress):
-            final_ingress.append({"service": "http_status:404"})
+    final_comparable = {_ingress_to_comparable(rule) for rule in final_ingress}
+    current_comparable = {_ingress_to_comparable(rule) for rule in current_api_ingress_rules}
+    if current_comparable == final_comparable and len(current_api_ingress_rules) == len(final_ingress):
+        logging.info(f"Tunnel {target_tunnel_id}: Cloudflare ingress already up to date.")
+        return True
 
-        final_comparable = {_ingress_to_comparable(rule) for rule in final_ingress}
-        current_comparable = {_ingress_to_comparable(rule) for rule in current_api_ingress_rules}
-
-        if current_comparable == final_comparable and len(current_api_ingress_rules) == len(final_ingress):
-            logging.info(f"Tunnel {target_tunnel_id}: Cloudflare ingress already up to date.")
-            return True
-
-        logging.info(f"Tunnel {target_tunnel_id}: Updating Cloudflare ingress with {len(final_ingress)} entries (desired {len(desired_ingress)}, preserved {len(preserved_rules)})")
-
-        try:
-            account_id = current_app.config.get('CF_ACCOUNT_ID')
-            endpoint = f"/accounts/{account_id}/cfd_tunnel/{target_tunnel_id}/configurations"
-            cloudflare_api.cf_api_request("PUT", endpoint, json_data={"config": {"ingress": final_ingress}})
-            return True
-        except Exception as exc:
-            logging.error(f"Failed to update Cloudflare config for tunnel {target_tunnel_id}: {exc}", exc_info=True)
-            if target_tunnel_id == master_tunnel_id:
-                tunnel_state["error"] = f"Failed to set tunnel config: {exc}"
-            return False
+    logging.info(f"Tunnel {target_tunnel_id}: Updating Cloudflare ingress with {len(final_ingress)} entries (desired {len(desired_ingress)}, preserved {len(preserved_rules)})")
+    try:
+        account_id = current_app.config.get('CF_ACCOUNT_ID')
+        endpoint = f"/accounts/{account_id}/cfd_tunnel/{target_tunnel_id}/configurations"
+        cloudflare_api.cf_api_request("PUT", endpoint, json_data={"config": {"ingress": final_ingress}})
+        return True
+    except Exception as exc:
+        logging.error(f"Failed to update Cloudflare config for tunnel {target_tunnel_id}: {exc}", exc_info=True)
+        if target_tunnel_id == master_tunnel_id:
+            tunnel_state["error"] = f"Failed to set tunnel config: {exc}"
+        return False
 
 
 def _build_ingress_entries_for_tunnel(target_tunnel_id, master_tunnel_id):
     entries = []
+    now = datetime.now(timezone.utc)
     with state_lock:
         for rule_key, rule_details in managed_rules.items():
-            if rule_details.get("status") != "active":
+            if not is_rule_ingress_eligible(rule_details, now):
                 continue
             if not _rule_applies_to_tunnel(rule_details, target_tunnel_id, master_tunnel_id):
                 continue
             entry = _build_ingress_entry_from_rule(rule_details)
             if entry:
                 entries.append(entry)
-    if target_tunnel_id:
-        agents_map = list_agents()
-        for agent_id, agent_details in agents_map.items():
-            if agent_details.get("assigned_tunnel_id") != target_tunnel_id:
-                continue
-            agent_rules = get_agent_rules(agent_id)
-            for rule_details in agent_rules.values():
-                if rule_details.get("status") != "active":
-                    continue
-                entry = _build_ingress_entry_from_rule(rule_details)
-                if entry:
-                    entries.append(entry)
     entries.append({"service": "http_status:404"})
     unique_entries = []
     seen = set()

dockflare/app/core/utils.py

@@ -17,6 +17,7 @@
 # dockflare/app/core/utils.py
 from app import config
 from app.core.container_name import build_cloudflared_container_name
+from app.core.zone_resolver import normalize_dns_name
 
 
 def get_rule_key(hostname, path):
@@ -24,6 +25,16 @@ def get_rule_key(hostname, path):
     return f"{hostname}|{path_str}"
 
 
+def get_source_rule_key(hostname, path):
+    """Return the canonical immutable identity derived from source labels."""
+    raw_hostname = str(hostname or "").strip()
+    wildcard = raw_hostname.startswith("*.")
+    normalized_hostname = normalize_dns_name(raw_hostname[2:] if wildcard else raw_hostname)
+    if wildcard:
+        normalized_hostname = f"*.{normalized_hostname}"
+    return get_rule_key(normalized_hostname, path)
+
+
 def get_label(labels, key_suffix, default=None):
     if config.CUSTOM_LABEL_PREFIX:
         custom_key = f"{config.CUSTOM_LABEL_PREFIX.rstrip('.')}.{key_suffix}"

dockflare/app/i18n/ch-barnduetsch.json

@@ -59,6 +59,8 @@
     "status.save_changes": "Änderige speichere",
     "status.rule_ui_override": "Regle via UI überschriebe",
     "status.ui_override": "UI-Override",
+    "status.tunnel_sync_pending": "Tunnel-Synchronisierung usstehend",
+    "status.tunnel_sync_pending_help": "D gwünschti Route isch lokal gspeicheret und wartet uf d Tunnel-Synchronisierung.",
     "status.docker_rule_overridden": "Die Docker-Regle isch via UI überschriebe worde u stimmt nüm mit de Container-Labels überei.",
     "status.ui_policy_override": "Die Richtlinie wird via UI verwaltet u überschrybt d Container-Labels.",
     "status.revert_to_docker_labels": "Uf Docker-Labels zrüggsetze",

dockflare/app/i18n/de.json

@@ -59,6 +59,8 @@
     "status.save_changes": "Änderungen speichern",
     "status.rule_ui_override": "Regel per UI überschrieben",
     "status.ui_override": "UI-Override",
+    "status.tunnel_sync_pending": "Tunnel-Synchronisierung ausstehend",
+    "status.tunnel_sync_pending_help": "Die gewünschte Route ist lokal gespeichert und wartet auf die Tunnel-Synchronisierung.",
     "status.docker_rule_overridden": "Diese Docker-Regel wurde per UI überschrieben und spiegelt nicht mehr die Container-Labels wider.",
     "status.ui_policy_override": "Diese Richtlinie wird per UI verwaltet und überschreibt die Container-Labels.",
     "status.revert_to_docker_labels": "Zu Docker-Labels zurücksetzen",

dockflare/app/i18n/en.json

@@ -11,6 +11,8 @@
     "common.edit": "Edit",
     "common.save": "Save",
     "common.copy": "Copy",
+    "common.copied": "Copied!",
+    "common.copy_failed": "Failed to copy text.",
     "common.loading": "Loading...",
     "common.actions": "Actions",
     "common.status": "Status",
@@ -59,6 +61,8 @@
     "status.save_changes": "Save Changes",
     "status.rule_ui_override": "Rule UI Override",
     "status.ui_override": "UI Override",
+    "status.tunnel_sync_pending": "Tunnel sync pending",
+    "status.tunnel_sync_pending_help": "The desired route is saved locally and awaiting tunnel synchronization.",
     "status.docker_rule_overridden": "This Docker rule has been overridden via UI and no longer reflects container labels.",
     "status.ui_policy_override": "This policy is managed by the UI and overrides container labels.",
     "status.revert_to_docker_labels": "Revert to Docker Labels",
@@ -160,6 +164,52 @@
     "settings.backup_restore": "Backup & Restore",
     "settings.security": "Security",
     "settings.oauth_authentication": "OAuth Authentication",
+    "settings.notifications": "Notifications",
+    "settings.notifications_description": "Send selected DockFlare lifecycle and failure alerts through one or more Apprise destinations.",
+    "settings.notification_status": "Status",
+    "settings.notification_unavailable": "Unavailable",
+    "settings.notification_disabled": "Disabled",
+    "settings.configured_destinations": "Configured destinations",
+    "settings.enable_notifications": "Enable notifications",
+    "settings.replacement_apprise_urls": "Replacement Apprise URLs",
+    "settings.replacement_apprise_urls_help": "Enter one URL per line. Leave blank to keep existing destinations. Saved URLs are never shown again.",
+    "settings.clear_notification_destinations": "Clear all configured destinations",
+    "settings.notification_cooldown": "Failure cooldown (seconds)",
+    "settings.notification_cooldown_help": "Suppresses repeated alerts for the same incident for 60 to 86400 seconds.",
+    "settings.notification_events": "Events",
+    "settings.notification_group_rules": "Rule lifecycle",
+    "settings.notification_group_failures": "Cloudflare and system failures",
+    "settings.notification_group_agents": "Agent lifecycle",
+    "settings.notification_group_tunnels": "Tunnel health",
+    "settings.notification_group_access_policies": "Access Policy management",
+    "settings.notification_event_rule_activated": "Rule activated",
+    "settings.notification_event_rule_restored": "Rule restored",
+    "settings.notification_event_rule_pending_deletion": "Rule pending deletion",
+    "settings.notification_event_rule_deleted": "Rule deleted",
+    "settings.notification_event_tunnel_failure": "Cloudflare tunnel update failure",
+    "settings.notification_event_dns_failure": "Cloudflare DNS failure",
+    "settings.notification_event_access_failure": "Cloudflare Access failure",
+    "settings.notification_event_docker_listener_failure": "Docker listener stopped",
+    "settings.notification_event_agent_offline": "Agent offline",
+    "settings.notification_event_agent_online": "Agent recovered",
+    "settings.notification_event_agent_enrolled": "Agent enrolled",
+    "settings.notification_event_agent_enrollment_failed": "Agent enrollment failed",
+    "settings.notification_event_agent_decommission_started": "Agent decommission started",
+    "settings.notification_event_agent_decommission_completed": "Agent decommission completed",
+    "settings.notification_event_agent_decommission_failed": "Agent decommission failed",
+    "settings.notification_event_agent_decommission_stalled": "Agent decommission timed out",
+    "settings.notification_event_tunnel_down": "Tunnel down",
+    "settings.notification_event_tunnel_recovered": "Tunnel recovered",
+    "settings.notification_event_access_policy_created": "Access Policy created",
+    "settings.notification_event_access_policy_updated": "Access Policy updated",
+    "settings.notification_event_access_policy_deleted": "Access Policy deleted",
+    "settings.save_notification_settings": "Save notification settings",
+    "settings.send_test_notification": "Send test notification",
+    "settings.notification_security_warning": "Apprise URLs often contain secrets and can make DockFlare connect to external or internal addresses. Configure only destinations you trust. Backups contain the encrypted configuration and its key and must be stored securely.",
+    "settings.test_notification_pending": "Sending test notification...",
+    "settings.test_notification_success": "Test notification delivered successfully.",
+    "settings.test_notification_failed": "Test notification failed. Check DockFlare logs.",
+    "settings.test_notification_timeout": "Test notification is still pending. Check DockFlare logs.",
     "settings.tunnel_agent_status": "Tunnel & Agent Status",
     "settings.check_version": "Check version",
     "settings.check_version_title": "Check DockFlare version",
@@ -407,6 +457,76 @@
     "agents.deploy_compose_snippet": "Compose Snippet",
     "agents.deploy_quick_desc": "Copy this script and paste it directly into your SSH session on the target server.",
     "agents.deploy_compose_desc": "Save as <code>docker-compose.yml</code>, ensure <code>cloudflare-net</code> network exists, then run <code>docker compose up -d</code>.",
+    "agents.decommission_title": "Decommission Agent",
+    "agents.decommission_action": "Decommission Agent",
+    "agents.decommission_view": "View decommission progress",
+    "agents.decommissioning": "Decommissioning",
+    "agents.decommission_warning": "Published services will become unavailable. Containers are stopped remotely but must be removed manually.",
+    "agents.decommission_contact": "Contact Agent",
+    "agents.decommission_stop_tunnel": "Stop tunnel",
+    "agents.decommission_cloudflare": "Clean resources",
+    "agents.decommission_schedule_agent": "Schedule Agent stop",
+    "agents.decommission_revoke_access": "Revoke Agent access",
+    "agents.decommission_complete": "Complete",
+    "agents.decommission_state": "Current state",
+    "agents.decommission_ready": "Review the impact, then start the decommission operation.",
+    "agents.decommission_resume": "Resuming the saved Agent decommission operation.",
+    "agents.decommission_online": "Agent is online",
+    "agents.decommission_online_short": "Online",
+    "agents.decommission_offline": "Agent is offline; cleanup will wait unless you explicitly force Master cleanup",
+    "agents.decommission_offline_short": "Offline",
+    "agents.decommission_unknown_agent": "Agent",
+    "agents.decommission_confirm": "Decommission this Agent? DockFlare will first stop its tunnel, clean only safely owned Cloudflare resources, revoke its key, and schedule the Agent container to stop.",
+    "agents.decommission_retry": "Retry current step",
+    "agents.decommission_force": "Force Master cleanup",
+    "agents.decommission_force_confirm": "Force Master-side cleanup without confirmed remote shutdown? Remote containers may remain running and must be cleaned on the Agent host.",
+    "agents.decommission_host_responsibility": "Run these commands on the Agent host. DockFlare does not execute them remotely and cannot verify their result.",
+    "agents.decommission_confirm_data": "I understand this deletes the Agent data volume and its persisted identity.",
+    "agents.decommission_files_warning": "Run the deployment-directory command only after the Docker cleanup finishes successfully. Skipping it leaves credential-bearing deployment files on the server.",
+    "agents.decommission_confirm_files": "I understand this permanently deletes $HOME/dockflare-agent and all files inside it.",
+    "agents.decommission_network_preserved": "The external cloudflare-net network is preserved because other containers may still use it.",
+    "agents.decommission_tunnel_disposition": "Tunnel ownership action",
+    "agents.decommission_last_contact": "Last contact",
+    "agents.decommission_assigned_tunnel": "Assigned tunnel",
+    "agents.decommission_remote_tunnel_action": "Remote tunnel-container action",
+    "agents.decommission_remote_agent_action": "Remote Agent-container action",
+    "agents.decommission_container_names": "Containers requiring host removal",
+    "agents.decommission_preserved_network": "Preserved external network",
+    "agents.decommission_stop_only": "stop only; never delete remotely",
+    "agents.decommission_stop_scheduled": "stop will be scheduled after final acknowledgement",
+    "agents.decommission_manual_required": "manual host stop is required",
+    "agents.decommission_unknown": "unknown; preserve by default",
+    "agents.decommission_reverse_deploy_notice": "After completion, run the provided reverse-deploy commands on the Agent host. The external cloudflare-net network is preserved.",
+    "agents.decommission_completed_message": "Agent decommissioned. The tunnel container was stopped, Cloudflare resources were cleaned according to ownership, Agent access was revoked, and the Agent container stop was scheduled. Complete host cleanup with the reverse-deploy steps below.",
+    "agents.decommission_manual_message": "Master cleanup completed and Agent access was revoked, but the Agent could not schedule its own stop. The remote Agent container may still be running and requires the host cleanup steps below.",
+    "agents.decommission_forced_message": "Master cleanup completed without confirmed remote shutdown. Remote containers may still be running and require the host cleanup steps below.",
+    "agents.decommission_waiting_prepare": "Waiting for the Agent to persist its safety marker and stop the tunnel container.",
+    "agents.decommission_contacting_and_stopping": "Contacting the Agent and waiting for the tunnel to stop…",
+    "agents.decommission_working_hint": "Operation in progress — this may take up to 3 minutes.",
+    "agents.decommission_remote_prepared": "The Agent is prepared and the tunnel container is stopped or absent.",
+    "agents.decommission_cleaning_master": "Cleaning safely owned Cloudflare and DockFlare resources.",
+    "agents.decommission_waiting_finalize": "Waiting for the Agent to acknowledge final shutdown scheduling.",
+    "agents.decommission_shutdown_scheduled": "Agent shutdown was acknowledged; revoking Agent access.",
+    "agents.decommission_prepare_failed": "The Agent could not complete remote preparation.",
+    "agents.decommission_cleanup_failed": "Master-side cleanup stopped safely and can be retried.",
+    "agents.decommission_finalize_failed": "Final Agent shutdown processing could not be completed.",
+    "agents.decommission_force_failed": "Forced Master cleanup stopped safely and can be retried.",
+    "agents.decommission_timed_out": "The Agent did not acknowledge the current step before its deadline.",
+    "agents.decommission_state_unknown": "The decommission operation is in progress.",
+    "agents.decommission_error_agent_timeout": "Check Agent connectivity, retry, or explicitly force Master cleanup.",
+    "agents.decommission_error_tombstone": "The Agent could not persist its local decommission marker.",
+    "agents.decommission_error_tunnel_stop": "The managed tunnel container could not be confirmed stopped.",
+    "agents.decommission_error_access": "A Cloudflare Access application could not be cleaned.",
+    "agents.decommission_error_dns": "A safely owned DNS record could not be cleaned.",
+    "agents.decommission_error_tunnel": "The exclusively owned Cloudflare tunnel could not be cleaned.",
+    "agents.decommission_error_tunnel_sync": "The preserved tunnel configuration could not be synchronized.",
+    "agents.decommission_error_persistence": "DockFlare could not persist the lifecycle transition.",
+    "agents.decommission_error_key": "Agent access could not be revoked.",
+    "agents.decommission_error_generic": "Review the server logs before retrying.",
+    "agents.decommission_disposition_delete": "delete the exclusively created tunnel",
+    "agents.decommission_disposition_shared": "preserve the shared tunnel",
+    "agents.decommission_disposition_adopted": "preserve the adopted tunnel",
+    "agents.decommission_disposition_unknown": "ownership unknown; preserve the tunnel",
     "setup.title": "DockFlare Setup",
     "setup.step1.create_admin": "Create Admin User",
     "setup.step1.final_step": "Final Step: Create Admin User",
@@ -700,6 +820,11 @@
     "flash.setup.settings_confirmed": "Settings confirmed. Please create an admin user to continue.",
     "flash.setup.required_fields_missing": "Warning: Missing required fields (CF_API_TOKEN or CF_ACCOUNT_ID). You will not be able to proceed.",
     "flash.setup.setup_complete": "Setup complete! Please log in to continue.",
+    "flash.notification_settings_invalid": "Notification settings are invalid.",
+    "flash.notification_settings_save_failed": "Failed to save notification settings.",
+    "flash.notification_destination_invalid": "Invalid Apprise destination: {detail}.",
+    "flash.notification_destination_required": "At least one Apprise destination is required before notifications can be enabled.",
+    "flash.notification_settings_saved": "Notification settings saved.",
     "form.setup.username": "Username",
     "form.setup.password": "Password",
     "form.setup.confirm_password": "Confirm Password",

dockflare/app/i18n/es.json

@@ -59,6 +59,8 @@
     "status.save_changes": "Guardar cambios",
     "status.rule_ui_override": "Regla sobrescrita desde la interfaz",
     "status.ui_override": "Sobrescritura de UI",
+    "status.tunnel_sync_pending": "Sincronización del túnel pendiente",
+    "status.tunnel_sync_pending_help": "La ruta deseada está guardada localmente y espera la sincronización del túnel.",
     "status.docker_rule_overridden": "Esta regla de Docker se ha sobrescrito desde la interfaz y ya no refleja las etiquetas del contenedor.",
     "status.ui_policy_override": "Esta política se gestiona desde la interfaz y sobrescribe las etiquetas del contenedor.",
     "status.revert_to_docker_labels": "Volver a las etiquetas de Docker",

dockflare/app/i18n/fr.json

@@ -59,6 +59,8 @@
     "status.save_changes": "Enregistrer les modifications",
     "status.rule_ui_override": "Règle remplacée via l'interface",
     "status.ui_override": "Remplacement UI",
+    "status.tunnel_sync_pending": "Synchronisation du tunnel en attente",
+    "status.tunnel_sync_pending_help": "La route souhaitée est enregistrée localement et attend la synchronisation du tunnel.",
     "status.docker_rule_overridden": "Cette règle Docker a été remplacée via l'interface et ne reflète plus les labels du conteneur.",
     "status.ui_policy_override": "Cette politique est gérée via l'interface et remplace les labels du conteneur.",
     "status.revert_to_docker_labels": "Revenir aux labels Docker",

dockflare/app/i18n/id.json

@@ -59,6 +59,8 @@
     "status.save_changes": "Simpan Perubahan",
     "status.rule_ui_override": "Rule UI Override",
     "status.ui_override": "UI Override",
+    "status.tunnel_sync_pending": "Sinkronisasi tunnel tertunda",
+    "status.tunnel_sync_pending_help": "Rute yang diinginkan disimpan secara lokal dan menunggu sinkronisasi tunnel.",
     "status.docker_rule_overridden": "Docker rule ini telah dioverride melalui UI dan tidak lagi mencerminkan label container.",
     "status.ui_policy_override": "Policy ini dikelola melalui UI dan mengoverride label container.",
     "status.revert_to_docker_labels": "Kembalikan ke Docker Labels",

dockflare/app/i18n/it.json

@@ -59,6 +59,8 @@
     "status.save_changes": "Salva modifiche",
     "status.rule_ui_override": "Regola sovrascritta dall'interfaccia",
     "status.ui_override": "Override UI",
+    "status.tunnel_sync_pending": "Sincronizzazione tunnel in sospeso",
+    "status.tunnel_sync_pending_help": "La rotta desiderata è salvata localmente e attende la sincronizzazione del tunnel.",
     "status.docker_rule_overridden": "Questa regola Docker è stata sovrascritta dall'interfaccia e non riflette più le etichette del container.",
     "status.ui_policy_override": "Questo criterio è gestito dall'interfaccia e sovrascrive le etichette del container.",
     "status.revert_to_docker_labels": "Ripristina etichette Docker",

dockflare/app/i18n/ja.json

@@ -59,6 +59,8 @@
     "status.save_changes": "変更を保存",
     "status.rule_ui_override": "UI によるルール上書き",
     "status.ui_override": "UI 上書き",
+    "status.tunnel_sync_pending": "トンネル同期を保留中",
+    "status.tunnel_sync_pending_help": "目的のルートはローカルに保存され、トンネルの同期を待っています。",
     "status.docker_rule_overridden": "この Docker ルールは UI から上書きされており、コンテナ ラベルを反映していません。",
     "status.ui_policy_override": "このポリシーは UI で管理されており、コンテナ ラベルを上書きします。",
     "status.revert_to_docker_labels": "Docker ラベルに戻す",

dockflare/app/i18n/pl.json

@@ -59,6 +59,8 @@
     "status.save_changes": "Zapisz zmiany",
     "status.rule_ui_override": "Reguła nadpisana w UI",
     "status.ui_override": "Nadpisanie UI",
+    "status.tunnel_sync_pending": "Oczekiwanie na synchronizację tunelu",
+    "status.tunnel_sync_pending_help": "Docelowa trasa jest zapisana lokalnie i oczekuje na synchronizację tunelu.",
     "status.docker_rule_overridden": "Ta reguła Dockera została nadpisana w interfejsie i nie odzwierciedla już etykiet kontenera.",
     "status.ui_policy_override": "Ta polityka jest zarządzana z poziomu interfejsu i nadpisuje etykiety kontenera.",
     "status.revert_to_docker_labels": "Przywróć etykiety Dockera",

dockflare/app/i18n/sk.json

@@ -59,6 +59,8 @@
   "status.save_changes": "Uložiť zmeny",
   "status.rule_ui_override": "Pravidlo prepísané v UI",
   "status.ui_override": "Prepísané v UI",
+  "status.tunnel_sync_pending": "Čaká sa na synchronizáciu tunela",
+  "status.tunnel_sync_pending_help": "Požadovaná trasa je uložená lokálne a čaká na synchronizáciu tunela.",
   "status.docker_rule_overridden": "Toto pravidlo Docker bolo prepísané v UI a už nezodpovedá labelom kontajnera.",
   "status.ui_policy_override": "Túto politiku spravuje UI a má prednosť pred labelmi kontajnera.",
   "status.revert_to_docker_labels": "Obnoviť podľa Docker labelov",

dockflare/app/i18n/zh.json

@@ -59,6 +59,8 @@
     "status.save_changes": "保存更改",
     "status.rule_ui_override": "规则已被界面覆盖",
     "status.ui_override": "界面覆盖",
+    "status.tunnel_sync_pending": "隧道同步待处理",
+    "status.tunnel_sync_pending_help": "所需路由已保存在本地，正在等待隧道同步。",
     "status.docker_rule_overridden": "此 Docker 规则已通过界面覆盖，不再反映容器标签。",
     "status.ui_policy_override": "此策略由界面管理，并覆盖容器标签。",
     "status.revert_to_docker_labels": "还原为 Docker 标签",

dockflare/app/main.py

@@ -23,6 +23,7 @@ import sys
 import os
 import json
 import secrets
+import copy
 from cryptography.fernet import Fernet
 
 from app import app, docker_client, tunnel_state, cloudflared_agent_state, config
@@ -35,11 +36,14 @@ from app.core.tunnel_manager import (
 )
 from app.core.docker_handler import start_event_listeners, process_container_start
 from app.core.reconciler import cleanup_expired_rules, reconcile_state_threaded
+from app.core.agent_decommission import timeout_worker as agent_decommission_timeout_worker
+from app.core import notification_manager
 
 stop_event = threading.Event()
 background_threads_list = []
 agent_status_updater_thread = None
 main_initialization_thread = None
+notification_health_thread = None
 
 
 def _restart_if_requested():
@@ -54,7 +58,7 @@ def _restart_if_requested():
         sys.exit(0)
 
 def run_all_background_tasks():
-    global background_threads_list, agent_status_updater_thread 
+    global background_threads_list, agent_status_updater_thread, notification_health_thread
     
     threads_to_start = []
     if not docker_client:
@@ -85,6 +89,14 @@ def run_all_background_tasks():
         logging.info("Starting periodic agent status updater thread...")
         agent_status_updater_thread = threading.Thread(target=periodic_agent_status_updater, name="AgentStatusUpdater", daemon=True)
         threads_to_start.append(agent_status_updater_thread)
+
+    if notification_health_thread is None or not notification_health_thread.is_alive():
+        notification_health_thread = threading.Thread(
+            target=periodic_notification_health_monitor,
+            name="NotificationHealthMonitor",
+            daemon=True,
+        )
+        threads_to_start.append(notification_health_thread)
     
     for t in threads_to_start:
         t.start()
@@ -107,12 +119,62 @@ def periodic_agent_status_updater():
         stop_event.wait(config.AGENT_STATUS_UPDATE_INTERVAL_SECONDS)
     logging.info("Periodic agent status updater task stopped.")
 
+
+def periodic_notification_health_monitor():
+    from app.core.state_manager import agents, agent_decommissions, state_lock
+
+    logging.info("Notification health monitor starting...")
+    while not stop_event.is_set():
+        try:
+            with state_lock:
+                agents_snapshot = copy.deepcopy(agents)
+                decommissioning_ids = {
+                    operation.get("agent_id")
+                    for operation in agent_decommissions.values()
+                    if operation.get("state") not in {"completed", "failed", "cancelled"}
+                }
+                local_status = cloudflared_agent_state.get("container_status")
+                local_tunnel_id = tunnel_state.get("id") or config.EXTERNAL_TUNNEL_ID
+                local_tunnel_name = tunnel_state.get("name")
+            notification_manager.check_agent_health(
+                agents_snapshot,
+                config.AGENT_HEARTBEAT_TIMEOUT,
+                decommissioning_ids=decommissioning_ids,
+            )
+            if not config.USE_EXTERNAL_CLOUDFLARED:
+                notification_manager.check_tunnel_health(
+                    local_tunnel_id,
+                    local_status,
+                    {"tunnel_id": str(local_tunnel_id or "")[:12], "tunnel_name": local_tunnel_name or ""},
+                )
+            for agent_id, agent in agents_snapshot.items():
+                assigned_tunnel_id = agent.get("assigned_tunnel_id")
+                tunnel_status = (agent.get("tunnel_status") or {}).get("status")
+                if assigned_tunnel_id and tunnel_status:
+                    notification_manager.check_tunnel_health(
+                        assigned_tunnel_id,
+                        tunnel_status,
+                        {
+                            "tunnel_id": str(assigned_tunnel_id)[:12],
+                            "tunnel_name": agent.get("assigned_tunnel_name") or "",
+                            "agent_id": str(agent_id)[:12],
+                            "agent_name": agent.get("display_name") or "",
+                        },
+                        intentional=agent_id in decommissioning_ids,
+                    )
+        except Exception:
+            logging.exception("Notification health monitor pass failed.")
+        stop_event.wait(config.NOTIFICATION_HEALTH_INTERVAL_SECONDS)
+    logging.info("Notification health monitor stopped.")
+
 def start_core_services():
     global background_threads_list 
 
     logging.info("Core services initialization process started.")
     if not docker_client:
         logging.error("Docker client unavailable. Critical functionalities will be affected.")
+        notification_manager.end_bootstrap()
+        run_all_background_tasks()
         return
 
     initialize_tunnel()
@@ -245,6 +307,7 @@ def start_core_services():
         except Exception as e:
             logging.debug(f"Could not refresh email sending status on startup: {e}")
 
+    notification_manager.end_bootstrap()
     run_all_background_tasks()
 
 def perform_initial_setup_and_tasks():
@@ -317,6 +380,18 @@ def main_application_entrypoint():
 
     load_state()
     logging.info("Initial state loading from file complete.")
+    notification_manager.configure(config.NOTIFICATION_CONFIG)
+    notification_manager.begin_bootstrap()
+    notification_worker = notification_manager.start(stop_event)
+    background_threads_list.append(notification_worker)
+    decommission_timeout_thread = threading.Thread(
+        target=agent_decommission_timeout_worker,
+        args=(stop_event,),
+        name="AgentDecommissionTimeout",
+        daemon=True,
+    )
+    decommission_timeout_thread.start()
+    background_threads_list.append(decommission_timeout_thread)
     
     ensure_default_bypass_policy(flask_app=app)
     logging.info("Default bypass policy initialization complete.")
@@ -416,6 +491,7 @@ def main_application_entrypoint():
     finally:
         logging.info("Shutdown sequence initiated...")
         stop_event.set() 
+        notification_manager.stop(timeout_seconds=5)
         
         if main_initialization_thread and main_initialization_thread.is_alive():
             logging.info("Waiting for main initialization thread to complete (timeout 15s)...")

dockflare/app/templates/agents.html

@@ -248,6 +248,100 @@
   <form method="dialog" class="modal-backdrop"><button>{{ t('common.close') }}</button></form>
 </dialog>
 
+<dialog id="modal-decommission-agent" class="modal">
+  <div class="modal-box decommission-modal-box w-11/12 max-w-5xl">
+    <div class="decommission-modal-header">
+      <h3 class="font-bold text-lg">{{ t('agents.decommission_title') }}</h3>
+      <div class="flex items-center justify-between gap-3 py-3">
+        <p id="decommission-summary" class="min-w-0 text-sm font-medium"></p>
+        <span id="decommission-agent-status" class="badge badge-sm hidden shrink-0"></span>
+      </div>
+    </div>
+    <div id="decommission-preflight" class="hidden mb-4 rounded-md border border-base-300 bg-base-200/50 p-4">
+      <dl class="grid grid-cols-1 gap-3 sm:grid-cols-2 sm:gap-6">
+        <div>
+          <dt class="text-xs uppercase tracking-wide text-base-content/60">{{ t('agents.decommission_last_contact') }}</dt>
+          <dd id="decommission-last-contact" class="mt-1 text-sm font-medium"></dd>
+        </div>
+        <div>
+          <dt class="text-xs uppercase tracking-wide text-base-content/60">{{ t('agents.decommission_assigned_tunnel') }}</dt>
+          <dd id="decommission-assigned-tunnel" class="mt-1 text-sm font-medium break-all"></dd>
+        </div>
+        <div>
+          <dt class="text-xs uppercase tracking-wide text-base-content/60">{{ t('agents.decommission_tunnel_disposition') }}</dt>
+          <dd id="decommission-tunnel-disposition" class="mt-1 text-sm font-medium"></dd>
+        </div>
+        <div>
+          <dt class="text-xs uppercase tracking-wide text-base-content/60">{{ t('agents.decommission_remote_tunnel_action') }}</dt>
+          <dd id="decommission-tunnel-action" class="mt-1 text-sm font-medium"></dd>
+        </div>
+        <div>
+          <dt class="text-xs uppercase tracking-wide text-base-content/60">{{ t('agents.decommission_remote_agent_action') }}</dt>
+          <dd id="decommission-agent-action" class="mt-1 text-sm font-medium"></dd>
+        </div>
+        <div>
+          <dt class="text-xs uppercase tracking-wide text-base-content/60">{{ t('agents.decommission_container_names') }}</dt>
+          <dd id="decommission-container-names" class="mt-1 text-sm font-mono break-all"></dd>
+        </div>
+        <div>
+          <dt class="text-xs uppercase tracking-wide text-base-content/60">{{ t('agents.decommission_preserved_network') }}</dt>
+          <dd id="decommission-preserved-network" class="mt-1 text-sm font-mono"></dd>
+        </div>
+      </dl>
+      <p class="mt-4 border-t border-base-300 pt-2 text-xs text-base-content/70">{{ t('agents.decommission_reverse_deploy_notice') }}</p>
+    </div>
+    <div class="alert alert-warning mb-4">
+      <span>{{ t('agents.decommission_warning') }}</span>
+    </div>
+    <ol id="decommission-progress" class="steps steps-vertical sm:steps-horizontal hidden w-full text-xs mb-4">
+      <li class="step" data-phase="waiting_for_prepare">{{ t('agents.decommission_contact') }}</li>
+      <li class="step" data-phase="remote_prepared">{{ t('agents.decommission_stop_tunnel') }}</li>
+      <li class="step" data-phase="master_cleanup">{{ t('agents.decommission_cloudflare') }}</li>
+      <li class="step" data-phase="waiting_for_finalize">{{ t('agents.decommission_schedule_agent') }}</li>
+      <li class="step" data-phase="shutdown_scheduled">{{ t('agents.decommission_revoke_access') }}</li>
+      <li class="step" data-phase="completed">{{ t('agents.decommission_complete') }}</li>
+    </ol>
+    <div id="decommission-status" class="mb-4" role="status" aria-live="polite" aria-atomic="true">
+      <div id="decommission-working" class="decommission-working-panel hidden items-center gap-3 rounded-md border p-3">
+        <span class="loading loading-spinner decommission-working-spinner text-primary shrink-0" aria-hidden="true"></span>
+        <div>
+          <p id="decommission-working-message" class="text-sm font-medium"></p>
+          <p class="mt-1 text-xs text-base-content/60">{{ t('agents.decommission_working_hint') }}</p>
+        </div>
+      </div>
+      <div id="decommission-message" class="text-sm"></div>
+    </div>
+    <div id="decommission-cleanup" class="hidden space-y-4">
+      <div class="alert alert-info"><span>{{ t('agents.decommission_host_responsibility') }}</span></div>
+      <label class="label cursor-pointer justify-start gap-3">
+        <input id="confirm-agent-data-delete" type="checkbox" class="checkbox checkbox-warning">
+        <span class="label-text">{{ t('agents.decommission_confirm_data') }}</span>
+      </label>
+      <div class="relative bg-base-200 p-3 rounded-md">
+        <pre id="decommission-docker-command" class="font-mono text-xs whitespace-pre-wrap break-all"></pre>
+        <button id="copy-decommission-docker" class="btn btn-ghost btn-sm absolute top-1 right-1" disabled>{{ t('common.copy') }}</button>
+      </div>
+      <div class="alert alert-error"><span>{{ t('agents.decommission_files_warning') }}</span></div>
+      <label class="label cursor-pointer justify-start gap-3">
+        <input id="confirm-agent-files-delete" type="checkbox" class="checkbox checkbox-error">
+        <span class="label-text">{{ t('agents.decommission_confirm_files') }}</span>
+      </label>
+      <div class="relative bg-base-200 p-3 rounded-md">
+        <pre id="decommission-files-command" class="font-mono text-xs whitespace-pre-wrap break-all"></pre>
+        <button id="copy-decommission-files" class="btn btn-ghost btn-sm absolute top-1 right-1" disabled>{{ t('common.copy') }}</button>
+      </div>
+      <p class="text-xs text-base-content/60">{{ t('agents.decommission_network_preserved') }}</p>
+    </div>
+    <div class="modal-action decommission-modal-actions">
+      <button id="btn-start-decommission" class="btn btn-error">{{ t('agents.decommission_action') }}</button>
+      <button id="btn-retry-decommission" class="btn btn-warning hidden">{{ t('agents.decommission_retry') }}</button>
+      <button id="btn-force-decommission" class="btn btn-error btn-outline hidden">{{ t('agents.decommission_force') }}</button>
+      <button type="button" class="btn btn-ghost" onclick="document.getElementById('modal-decommission-agent').close()">{{ t('common.cancel') }}</button>
+    </div>
+  </div>
+  <form method="dialog" class="modal-backdrop"><button>{{ t('common.close') }}</button></form>
+</dialog>
+
 <dialog id="modal-deploy-agent" class="modal">
   <div class="modal-box w-11/12 max-w-3xl">
     <h3 class="font-bold text-lg">{{ t('agents.deploy_agent') }}</h3>
@@ -436,6 +530,51 @@
   vertical-align: top !important;
 }
 
+.decommission-modal-box {
+  max-height: calc(100vh - 2rem);
+  overflow-y: auto;
+}
+
+.decommission-modal-header {
+  position: sticky;
+  top: -1.5rem;
+  z-index: 10;
+  margin: -1.5rem -1.5rem 0;
+  padding: 1.5rem 1.5rem 0;
+  background: var(--fallback-b1, oklch(var(--b1) / 1));
+}
+
+.decommission-modal-actions {
+  position: sticky;
+  bottom: -1.5rem;
+  z-index: 10;
+  margin-right: -1.5rem;
+  margin-bottom: -1.5rem;
+  margin-left: -1.5rem;
+  padding: 1rem 1.5rem 1.5rem;
+  border-top: 1px solid var(--fallback-b3, oklch(var(--b3) / 1));
+  background: var(--fallback-b1, oklch(var(--b1) / 1));
+}
+
+.decommission-working-panel {
+  border-color: oklch(var(--p) / 0.3);
+  background: oklch(var(--p) / 0.1);
+}
+
+.decommission-working-spinner {
+  width: 1.25rem;
+  height: 1.25rem;
+}
+
+#decommission-progress .decommission-active-step::after,
+#decommission-progress.decommission-preparing .step:nth-child(2)::after {
+  animation: decommission-step-pulse 1.4s ease-in-out infinite;
+}
+
+#decommission-progress.decommission-preparing .step:nth-child(2)::before {
+  animation: decommission-connector-pulse 1.4s ease-in-out infinite;
+}
+
 /* Animations */
 @keyframes led-pulse-critical {
   0% {
@@ -474,6 +613,25 @@
   50% { background-color: #065f46; }
   100% { background-color: #1f2937; }
 }
+
+@keyframes decommission-step-pulse {
+  0%, 100% { box-shadow: 0 0 0 0 oklch(var(--p) / 0.4); }
+  50% { box-shadow: 0 0 0 0.45rem oklch(var(--p) / 0); }
+}
+
+@keyframes decommission-connector-pulse {
+  0%, 100% { opacity: 0.35; }
+  50% { opacity: 1; }
+}
+
+@media (prefers-reduced-motion: reduce) {
+  #decommission-progress .decommission-active-step::after,
+  #decommission-progress.decommission-preparing .step:nth-child(2)::after,
+  #decommission-progress.decommission-preparing .step:nth-child(2)::before,
+  #decommission-working .loading {
+    animation: none;
+  }
+}
 </style>
 
 <script>
@@ -642,13 +800,12 @@
 
   const copyToClipboard = (text, buttonElement) => {
     navigator.clipboard.writeText(text).then(() => {
-      const originalText = buttonElement.innerHTML;
-      buttonElement.innerHTML = 'Copied!';
+      buttonElement.innerHTML = t('common.copied');
       setTimeout(() => {
-        buttonElement.innerHTML = 'Copy';
+        buttonElement.innerHTML = t('common.copy');
       }, 2000);
-    }, async (err) => {
-      await dfAlert('Failed to copy text: ' + err, 'Error');
+    }, async () => {
+      await dfAlert(t('common.copy_failed'), t('js.text.error'));
     });
   };
 
@@ -671,7 +828,10 @@
 
       const enrollmentBadge = getStatusBadge(agent.status);
       const healthBadge = getStatusBadge(agent.health || (agent.online ? 'connected' : 'disconnected'));
-      const statusHtml = `${enrollmentBadge} ${healthBadge}`;
+      const decommissionBadge = agent.decommission_operation_id
+        ? `<span class="badge badge-warning badge-sm">${t('agents.decommissioning')}</span>`
+        : '';
+      const statusHtml = `${enrollmentBadge} ${healthBadge} ${decommissionBadge}`;
 
       const ledStripHtml = createLedStrip(agentId, agent.last_seen, agent.online);
       const na = '<span class="text-base-content/30">—</span>';
@@ -724,7 +884,8 @@
           <button class="btn btn-ghost btn-sm agent-action-trigger"
             data-agent-id="${agentId}"
             data-enrolled="${agent.status === 'enrolled'}"
-            data-has-tunnel="${!!agent.assigned_tunnel_name}">
+            data-has-tunnel="${!!agent.assigned_tunnel_name}"
+            data-decommission-operation="${agent.decommission_operation_id || ''}">
             <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-4 h-4">
               <path stroke-linecap="round" stroke-linejoin="round" d="M12 6.75a.75.75 0 110-1.5.75.75 0 010 1.5zM12 12.75a.75.75 0 110-1.5.75.75 0 010 1.5zM12 18.75a.75.75 0 110-1.5.75.75 0 010 1.5z" />
             </svg>
@@ -732,7 +893,9 @@
         </td>
       `;
 
-      if (agent.online === false) {
+      if (agent.decommission_operation_id) {
+        tr.style.borderLeft = '4px solid #f59e0b';
+      } else if (agent.online === false) {
         tr.style.backgroundColor = '#fff5f5';
         tr.style.borderLeft = '4px solid #f87171';
       }
@@ -968,6 +1131,224 @@
   setInterval(refreshData, 5000);
   setInterval(updateLedStrips, 1000); // Update LED strips every second
 
+  let currentDecommissionAgentId = null;
+  let currentDecommissionOperationId = null;
+  let currentDecommissionState = null;
+  let currentDecommissionAgentOnline = null;
+  let currentDecommissionPreview = null;
+  let decommissionPollTimer = null;
+
+  const decommissionPhaseOrder = {
+    waiting_for_prepare: 0,
+    prepare_failed: 0,
+    timed_out: 0,
+    remote_prepared: 1,
+    master_cleanup: 2,
+    cleanup_failed: 2,
+    waiting_for_finalize: 3,
+    finalize_failed: 3,
+    shutdown_scheduled: 4,
+    completed: 5,
+    forced_completed: 5,
+    force_failed: 3,
+  };
+
+  const decommissionMessageKeys = {
+    waiting_for_prepare: 'agents.decommission_waiting_prepare',
+    remote_prepared: 'agents.decommission_remote_prepared',
+    master_cleanup: 'agents.decommission_cleaning_master',
+    waiting_for_finalize: 'agents.decommission_waiting_finalize',
+    shutdown_scheduled: 'agents.decommission_shutdown_scheduled',
+    prepare_failed: 'agents.decommission_prepare_failed',
+    cleanup_failed: 'agents.decommission_cleanup_failed',
+    finalize_failed: 'agents.decommission_finalize_failed',
+    force_failed: 'agents.decommission_force_failed',
+    timed_out: 'agents.decommission_timed_out',
+  };
+
+  const decommissionErrorKeys = {
+    agent_timeout: 'agents.decommission_error_agent_timeout',
+    tombstone_persistence_failed: 'agents.decommission_error_tombstone',
+    tunnel_stop_failed: 'agents.decommission_error_tunnel_stop',
+    access_cleanup_failed: 'agents.decommission_error_access',
+    dns_cleanup_failed: 'agents.decommission_error_dns',
+    tunnel_cleanup_failed: 'agents.decommission_error_tunnel',
+    tunnel_sync_failed: 'agents.decommission_error_tunnel_sync',
+    persistence_failed: 'agents.decommission_error_persistence',
+    key_revocation_failed: 'agents.decommission_error_key',
+  };
+
+  const decommissionDispositionKeys = {
+    delete_exclusive: 'agents.decommission_disposition_delete',
+    preserve_shared: 'agents.decommission_disposition_shared',
+    preserve_adopted: 'agents.decommission_disposition_adopted',
+    unknown: 'agents.decommission_disposition_unknown',
+  };
+
+  const resetDecommissionModal = () => {
+    document.getElementById('decommission-cleanup').classList.add('hidden');
+    document.getElementById('btn-start-decommission').classList.remove('hidden');
+    document.getElementById('btn-retry-decommission').classList.add('hidden');
+    document.getElementById('btn-force-decommission').classList.add('hidden');
+    document.getElementById('confirm-agent-data-delete').checked = false;
+    document.getElementById('confirm-agent-files-delete').checked = false;
+    document.getElementById('copy-decommission-docker').disabled = true;
+    document.getElementById('copy-decommission-files').disabled = true;
+    document.getElementById('decommission-docker-command').textContent = '';
+    document.getElementById('decommission-files-command').textContent = '';
+    document.getElementById('decommission-preflight').classList.add('hidden');
+    document.getElementById('decommission-progress').classList.add('hidden');
+    document.getElementById('decommission-progress').classList.remove('decommission-preparing');
+    document.getElementById('decommission-working').classList.add('hidden');
+    document.getElementById('decommission-working').classList.remove('flex');
+    document.getElementById('decommission-message').classList.remove('hidden');
+    document.getElementById('decommission-agent-status').classList.add('hidden');
+    currentDecommissionPreview = null;
+    document.querySelectorAll('#decommission-progress .step').forEach(step => {
+      step.classList.remove('step-primary', 'step-error', 'decommission-active-step');
+      step.removeAttribute('aria-current');
+      step.removeAttribute('data-content');
+    });
+  };
+
+  const renderDecommissionPreflight = (value) => {
+    currentDecommissionPreview = {
+      ...(currentDecommissionPreview || {}),
+      ...value,
+      resource_plan: {
+        ...(currentDecommissionPreview?.resource_plan || {}),
+        ...(value.resource_plan || {}),
+      },
+      remote_actions: {
+        ...(currentDecommissionPreview?.remote_actions || {}),
+        ...(value.remote_actions || {}),
+      },
+    };
+    const preview = currentDecommissionPreview;
+    const plan = preview.resource_plan || {};
+    const remoteActions = preview.remote_actions || {};
+    const hostPlan = preview.host_cleanup_plan || {};
+    const names = preview.container_names || [hostPlan.tunnel_container_name, hostPlan.agent_container_name].filter(Boolean);
+    const tunnelAction = t('agents.decommission_stop_only');
+    const agentActionValue = preview.remote_results?.agent_container || remoteActions.agent_container;
+    const agentAction = ['manual', 'manual_required'].includes(agentActionValue)
+      ? t('agents.decommission_manual_required')
+      : t('agents.decommission_stop_scheduled');
+    document.getElementById('decommission-last-contact').textContent = preview.last_seen
+      ? new Date(preview.last_seen).toLocaleString()
+      : t('agents.decommission_unknown');
+    document.getElementById('decommission-assigned-tunnel').textContent = preview.assigned_tunnel_name || t('agents.decommission_unknown');
+    document.getElementById('decommission-tunnel-disposition').textContent = t(
+      decommissionDispositionKeys[plan.tunnel_disposition] || 'agents.decommission_disposition_unknown'
+    );
+    document.getElementById('decommission-tunnel-action').textContent = tunnelAction;
+    document.getElementById('decommission-agent-action').textContent = agentAction;
+    document.getElementById('decommission-container-names').textContent = names.join(', ') || t('agents.decommission_unknown');
+    document.getElementById('decommission-preserved-network').textContent = (
+      preview.preserve_networks || hostPlan.preserve_networks || ['cloudflare-net']
+    ).join(', ');
+    const element = document.getElementById('decommission-preflight');
+    element.classList.remove('hidden');
+  };
+
+  const renderDecommissionOperation = (operation) => {
+    currentDecommissionState = operation.state;
+    currentDecommissionOperationId = operation.operation_id;
+    localStorage.setItem('dockflare-agent-decommission-operation', operation.operation_id);
+    document.getElementById('btn-start-decommission').classList.add('hidden');
+    document.getElementById('decommission-progress').classList.remove('hidden');
+    renderDecommissionPreflight(operation);
+    const phaseIndex = decommissionPhaseOrder[operation.state] ?? 0;
+    const complete = ['completed', 'forced_completed'].includes(operation.state);
+    const failed = operation.state.endsWith('failed') || operation.state === 'timed_out';
+    const running = !complete && !failed;
+    const progress = document.getElementById('decommission-progress');
+    progress.classList.toggle('decommission-preparing', operation.state === 'waiting_for_prepare');
+    document.querySelectorAll('#decommission-progress .step').forEach((step, index) => {
+      step.classList.toggle('step-primary', index <= phaseIndex);
+      step.classList.toggle('step-error', failed && index === phaseIndex);
+      step.classList.toggle('decommission-active-step', running && index === phaseIndex);
+      if (index < phaseIndex || (complete && index <= phaseIndex)) step.setAttribute('data-content', '✓');
+      else if (failed && index === phaseIndex) step.setAttribute('data-content', '!');
+      else step.removeAttribute('data-content');
+      if (index === phaseIndex) step.setAttribute('aria-current', 'step');
+      else step.removeAttribute('aria-current');
+    });
+    const message = document.getElementById('decommission-message');
+    const working = document.getElementById('decommission-working');
+    const workingMessage = document.getElementById('decommission-working-message');
+    if (operation.state === 'completed') {
+      message.textContent = operation.remote_results?.agent_container === 'manual_required'
+        ? t('agents.decommission_manual_message')
+        : t('agents.decommission_completed_message');
+    } else if (operation.state === 'forced_completed') {
+      message.textContent = t('agents.decommission_forced_message');
+    } else {
+      const stateText = t(decommissionMessageKeys[operation.state] || 'agents.decommission_state_unknown');
+      const errorText = operation.last_error_code
+        ? t(decommissionErrorKeys[operation.last_error_code] || 'agents.decommission_error_generic')
+        : '';
+      message.textContent = errorText ? `${stateText} ${errorText}` : stateText;
+    }
+    workingMessage.textContent = operation.state === 'waiting_for_prepare'
+      ? t('agents.decommission_contacting_and_stopping')
+      : message.textContent;
+    working.classList.toggle('hidden', !running);
+    working.classList.toggle('flex', running);
+    message.classList.toggle('hidden', running);
+    const agent = window.lastFetchedData?.agents?.[operation.agent_id];
+    if (agent) currentDecommissionAgentOnline = agent.online;
+    document.getElementById('btn-retry-decommission').classList.toggle('hidden', !failed);
+    const mayForceOffline = operation.state === 'waiting_for_prepare' && currentDecommissionAgentOnline === false;
+    document.getElementById('btn-force-decommission').classList.toggle('hidden', !failed && !mayForceOffline);
+    if (complete && operation.cleanup_commands) {
+      document.getElementById('decommission-cleanup').classList.remove('hidden');
+      document.getElementById('decommission-docker-command').textContent = operation.cleanup_commands.docker || '';
+      document.getElementById('decommission-files-command').textContent = operation.cleanup_commands.deployment_files || '';
+      if (decommissionPollTimer) {
+        clearInterval(decommissionPollTimer);
+        decommissionPollTimer = null;
+      }
+      fetchData();
+    }
+  };
+
+  const pollDecommission = async () => {
+    if (!currentDecommissionOperationId) return;
+    const result = await apiCall(`/api/v2/agent-decommissions/${currentDecommissionOperationId}`);
+    if (result && result.operation) renderDecommissionOperation(result.operation);
+  };
+
+  const setDecommissionActionPending = (button, pending) => {
+    button.disabled = pending;
+    button.setAttribute('aria-busy', pending ? 'true' : 'false');
+  };
+
+  const openDecommissionModal = async (agentId, operationId = null) => {
+    resetDecommissionModal();
+    currentDecommissionAgentId = agentId;
+    currentDecommissionOperationId = operationId;
+    const agent = window.lastFetchedData?.agents?.[agentId];
+    currentDecommissionAgentOnline = typeof agent?.online === 'boolean' ? agent.online : null;
+    const label = agent?.display_name || agentId || t('agents.decommission_unknown_agent');
+    const abbreviatedId = agentId ? ` (${agentId.slice(0, 8)}…)` : '';
+    document.getElementById('decommission-summary').textContent = `${label}${abbreviatedId}`;
+    const statusBadge = document.getElementById('decommission-agent-status');
+    statusBadge.textContent = agent?.online ? t('agents.decommission_online_short') : t('agents.decommission_offline_short');
+    statusBadge.className = `badge badge-sm shrink-0 ${agent?.online ? 'badge-success' : 'badge-warning'}`;
+    document.getElementById('decommission-message').textContent = agent?.online
+      ? t('agents.decommission_ready')
+      : t('agents.decommission_offline');
+    document.getElementById('modal-decommission-agent').showModal();
+    if (operationId) {
+      await pollDecommission();
+      if (!decommissionPollTimer) decommissionPollTimer = setInterval(pollDecommission, 2000);
+    } else {
+      const result = await apiCall(`/api/v2/agents/${agentId}/decommission-preview`);
+      if (result?.preview) renderDecommissionPreflight(result.preview);
+    }
+  };
+
   const openRenameModal = async (agentId) => {
     const data = window.lastFetchedData;
     if (!data || !data.agents || !data.agents[agentId]) {
@@ -986,6 +1367,69 @@
     loadCfTokenStatus();
     updateLedStrips();
 
+    const retainedOperation = localStorage.getItem('dockflare-agent-decommission-operation');
+    if (retainedOperation) {
+      resetDecommissionModal();
+      currentDecommissionOperationId = retainedOperation;
+      document.getElementById('decommission-summary').textContent = t('agents.decommission_resume');
+      document.getElementById('modal-decommission-agent').showModal();
+      pollDecommission();
+      decommissionPollTimer = setInterval(pollDecommission, 2000);
+    }
+
+    document.getElementById('modal-decommission-agent').addEventListener('close', () => {
+      if (['completed', 'forced_completed'].includes(currentDecommissionState)) {
+        localStorage.removeItem('dockflare-agent-decommission-operation');
+      }
+    });
+
+    document.getElementById('btn-start-decommission').addEventListener('click', async () => {
+      const confirmed = await dfConfirm(t('agents.decommission_confirm'), t('agents.decommission_title'));
+      if (!confirmed) return;
+      const button = document.getElementById('btn-start-decommission');
+      setDecommissionActionPending(button, true);
+      try {
+        const result = await apiCall(`/api/v2/agents/${currentDecommissionAgentId}/decommission`, 'POST');
+        if (result?.operation) {
+          renderDecommissionOperation(result.operation);
+          if (!decommissionPollTimer) decommissionPollTimer = setInterval(pollDecommission, 2000);
+        }
+      } finally {
+        setDecommissionActionPending(button, false);
+      }
+    });
+
+    document.getElementById('btn-retry-decommission').addEventListener('click', async () => {
+      const button = document.getElementById('btn-retry-decommission');
+      setDecommissionActionPending(button, true);
+      try {
+        const result = await apiCall(`/api/v2/agent-decommissions/${currentDecommissionOperationId}/retry`, 'POST');
+        if (result?.operation) renderDecommissionOperation(result.operation);
+      } finally {
+        setDecommissionActionPending(button, false);
+      }
+    });
+
+    document.getElementById('btn-force-decommission').addEventListener('click', async () => {
+      const confirmed = await dfConfirm(t('agents.decommission_force_confirm'), t('agents.decommission_force'));
+      if (!confirmed) return;
+      const result = await apiCall(`/api/v2/agent-decommissions/${currentDecommissionOperationId}/force`, 'POST');
+      if (result?.operation) renderDecommissionOperation(result.operation);
+    });
+
+    document.getElementById('confirm-agent-data-delete').addEventListener('change', (event) => {
+      document.getElementById('copy-decommission-docker').disabled = !event.target.checked;
+    });
+    document.getElementById('confirm-agent-files-delete').addEventListener('change', (event) => {
+      document.getElementById('copy-decommission-files').disabled = !event.target.checked;
+    });
+    document.getElementById('copy-decommission-docker').addEventListener('click', (event) => {
+      copyToClipboard(document.getElementById('decommission-docker-command').textContent, event.currentTarget);
+    });
+    document.getElementById('copy-decommission-files').addEventListener('click', (event) => {
+      copyToClipboard(document.getElementById('decommission-files-command').textContent, event.currentTarget);
+    });
+
     document.getElementById('btn-generate-key').addEventListener('click', () => {
       document.getElementById('generated-key-container').classList.add('hidden');
       document.getElementById('form-generate-key').reset();
@@ -1041,6 +1485,19 @@
       const agentId = btn.dataset.agentId;
       const isEnrolled = btn.dataset.enrolled === 'true';
       const hasTunnel = btn.dataset.hasTunnel === 'true';
+      const decommissionOperation = btn.dataset.decommissionOperation;
+
+      if (decommissionOperation) {
+        agentFloatingMenuList.innerHTML = `
+          <li><a class="btn-remove" data-agent-id="${agentId}" data-operation-id="${decommissionOperation}">${t('agents.decommission_view')}</a></li>
+        `;
+        agentFloatingMenu.classList.remove('hidden');
+        const rect = btn.getBoundingClientRect();
+        agentFloatingMenu.style.top = `${rect.bottom + 4}px`;
+        agentFloatingMenu.style.left = `${Math.max(4, rect.right - 208)}px`;
+        agentFloatingMenu._trigger = btn;
+        return;
+      }
 
       agentFloatingMenuList.innerHTML = `
         <li ${isEnrolled && hasTunnel ? 'class="disabled"' : ''}>
@@ -1078,7 +1535,7 @@
         <li>
           <a class="btn-remove text-error" data-agent-id="${agentId}">
             <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-4 h-4"><path stroke-linecap="round" stroke-linejoin="round" d="M14.74 9l-.346 9m-4.788 0L9.26 9m9.968-3.21c.342.052.682.107 1.022.166m-1.022-.165L18.16 19.673a2.25 2.25 0 01-2.244 2.077H8.084a2.25 2.25 0 01-2.244-2.077L4.772 5.79m14.456 0a48.108 48.108 0 00-3.478-.397m-12 .562c.34-.059.68-.114 1.022-.165m0 0a48.11 48.11 0 013.478-.397m7.5 0v-.916c0-1.18-.91-2.164-2.09-2.201a51.964 51.964 0 00-3.32 0c-1.18.037-2.09 1.022-2.09 2.201v.916m7.5 0a48.667 48.667 0 00-7.5 0" /></svg>
-            Remove Agent
+            ${t('agents.decommission_action')}
           </a>
         </li>
       `;
@@ -1144,15 +1601,10 @@
 
         document.getElementById('modal-enroll-agent').showModal();
       } else if (e.target.closest('.btn-remove')) {
-        const agentId = e.target.closest('.btn-remove').getAttribute('data-agent-id');
-        const confirmed = await dfConfirm(`Are you sure you want to remove agent "${agentId}"? This will also delete the associated Cloudflare tunnel and all related resources.`, 'Remove Agent');
-        if (confirmed) {
-          const success = await apiCall(`/api/v2/agents/${agentId}/remove`, 'POST');
-          if (success) {
-            await dfAlert('Agent removed successfully.', 'Success');
-            fetchData();
-          }
-        }
+        const button = e.target.closest('.btn-remove');
+        const agentId = button.getAttribute('data-agent-id');
+        const operationId = button.getAttribute('data-operation-id');
+        openDecommissionModal(agentId, operationId);
       } else if (e.target.closest('.btn-roll-key')) {
         const agentId = e.target.closest('.btn-roll-key').getAttribute('data-agent-id');
         const confirmed = await dfConfirm(`Are you sure you want to roll the API key for agent "${agentId}"? The agent will need to be updated with the new key.`, 'Roll API Key');
@@ -1371,4 +1823,4 @@
 
   });
 </script>
-{% endblock %}
+{% endblock %}

dockflare/app/templates/base.html

@@ -682,7 +682,7 @@
         };
     </script>
     <script src="{{ url_for('static', filename='js/tom-select.complete.min.js') }}"></script>
-    <script src="{{ url_for('static', filename='js/main.js') }}"></script>
+    <script src="{{ url_for('static', filename='js/main.js', v=main_js_version) }}"></script>
     {% block scripts %}{% endblock %}
 </body>
 </html>

dockflare/app/templates/docs/ch-barnduetsch/navigation.md

@@ -18,6 +18,8 @@ DockFlare isch e sälber hosteti Ingress-Lösig, wo dir d Verwaltig vo Cloudflar
     *   [Persistänte Status](State-Persistence.md)
 *   **Istellige**
     *   [Container-Labels](Container-Labels.md)
+    *   [Apprise Notifications (English)](Apprise-Notifications.md)
+    *   [Discord & Apprise API Setup (English)](Discord-Apprise-Notifications.md)
     *   [Identity Providers](Identity-Providers.md)
     *   [OAuth-Provider iirichte](OAuth-Provider-Setup.md)
 *   **Bruuch**

dockflare/app/templates/docs/de/navigation.md

@@ -20,6 +20,8 @@ Diese Dokumentation bietet einen umfassenden Überblick über DockFlare. Egal, o
     *   [Persistenz des Status](State-Persistence.md)
 *   **Konfiguration**
     *   [Container-Labels](Container-Labels.md)
+    *   [Apprise Notifications (English)](Apprise-Notifications.md)
+    *   [Discord & Apprise API Setup (English)](Discord-Apprise-Notifications.md)
     *   [Identitätsanbieter](Identity-Providers.md)
     *   [Einrichtung von OAuth-Anbietern](OAuth-Provider-Setup.md)
 *   **Benutzerhandbuch**

dockflare/app/templates/docs/en/Apprise-Notifications.md

@@ -0,0 +1,135 @@
+# Notifications with Apprise
+
+DockFlare can send operational notifications through [Apprise](https://github.com/caronc/apprise), an independent third-party open-source notification framework. Apprise supports Discord, Telegram, Slack, email, generic webhooks, and many other services.
+
+Notifications are optional and disabled by default. Existing DockFlare installations continue to operate normally when no destination is configured. Notification delivery is best-effort and never changes the result of the DockFlare operation that produced the event.
+
+## Choose a setup
+
+Apprise is already bundled inside the DockFlare container. You do not need to install Apprise or run another container to use supported destinations directly. A separately deployed Apprise API is optional.
+
+| Setup | Best for | What you enter in DockFlare |
+|---|---|---|
+| Direct destination | One or a few services and the simplest setup | A supported provider URL, such as the Discord webhook copied from Discord |
+| Self-hosted Apprise API | Central configuration shared by several applications | An API URL such as `apprise://apprise:8000/{CONFIG_KEY}` |
+
+For a complete Discord example covering both options, see [Discord Notifications and Apprise API](Discord-Apprise-Notifications.md).
+
+## Configure DockFlare
+
+1. Open **Settings**.
+2. Select **Notifications**.
+3. Paste one complete Apprise URL per line into **Replacement Apprise URLs**.
+4. Select the events you want to receive.
+5. Enable notifications.
+6. Select **Save notification settings**.
+7. Select **Send test notification** and wait for the delivery result.
+
+The replacement field is intentionally blank whenever the page loads:
+
+- Leaving it blank retains every currently saved destination.
+- Entering one or more URLs replaces the complete saved list.
+- Selecting **Clear all configured destinations** removes the saved list when the form is saved.
+- You must save a new destination before testing it.
+
+Complete destination URLs are never displayed again after saving. The settings page shows only a redacted scheme and destination count.
+
+## Available events
+
+Event controls are grouped by purpose.
+
+### Rule lifecycle
+
+- Rule activated
+- Rule restored
+- Rule pending deletion
+- Rule deleted
+
+### Cloudflare and system failures
+
+- Cloudflare tunnel update failure
+- Cloudflare DNS failure
+- Cloudflare Access failure
+- Docker listener stopped
+
+### Agent lifecycle
+
+- Agent enrolled
+- Agent enrollment failed
+- Agent offline
+- Agent recovered
+- Agent decommission started
+- Agent decommission completed
+- Agent decommission failed
+- Agent decommission timed out
+
+### Tunnel health
+
+- Tunnel down
+- Tunnel recovered
+
+### Access Policy management
+
+- Access Policy created
+- Access Policy updated
+- Access Policy deleted
+
+Access Policy lifecycle notifications report administrative changes only. Creating a policy does not assign it to a service. Services must still reference the policy through the UI, API, or Docker labels.
+
+Routine administrative events are opt-in. Recommended failure, recovery, enrollment, and terminal decommission events are enabled by default in a new notification configuration. You can change every event independently.
+
+## Cooldowns and recovery messages
+
+Repeated failure notifications for the same resource are limited by the configured failure cooldown. The default is 900 seconds.
+
+Recovery messages are correlated with incidents:
+
+- `Agent recovered` is sent only when DockFlare previously sent an offline alert for that Agent.
+- `Tunnel recovered` is sent only when DockFlare previously sent a down alert for that tunnel.
+- The initial healthy observation establishes a baseline and does not send a recovery message.
+
+Normal startup scans are suppressed to avoid replaying historical rule events. Expected Agent and tunnel health changes during decommissioning or an intentional restart are also suppressed.
+
+## Delivery behavior
+
+Notifications are placed on a bounded in-memory queue and delivered by a background worker. A slow or unavailable destination does not block Docker events, Agent processing, Cloudflare operations, reconciliation, or web requests.
+
+Keep these limitations in mind:
+
+- Queued messages can be lost if DockFlare exits before delivery.
+- Failed delivery is not automatically retried because some destinations may already have received the message.
+- Queue, cooldown, health-baseline, and recent test-job state are cleared by a restart.
+- A full queue drops new notification events rather than delaying DockFlare's primary work.
+
+## Security
+
+Treat every Apprise destination URL as a secret. URLs commonly contain webhook tokens, API keys, usernames, passwords, or a stored configuration key.
+
+- DockFlare stores destinations in its encrypted configuration, not in `state.json`.
+- Saved URLs are redacted from the UI, API responses, and logs.
+- Encrypted backups contain the notification configuration and its encryption key. Protect backups accordingly.
+- Configure only destinations and Apprise API servers you trust. A destination can make DockFlare connect to an internal or external address.
+- If a Discord webhook or another destination URL is exposed, rotate or delete it at the provider and save the replacement in DockFlare.
+
+## Troubleshooting
+
+| Problem | Check |
+|---|---|
+| Test button is disabled | Save at least one valid destination and enable notifications first. |
+| Destination validation fails | Compare the URL with the [Apprise service documentation](https://appriseit.com/services/). |
+| Test remains pending | Check DockFlare logs and confirm the notification worker is running. |
+| Delivery fails | Confirm the DockFlare container can resolve and reach the destination host and port. |
+| Apprise API works in a browser but not from DockFlare | Do not use `localhost` unless the API runs in the DockFlare container. Use a shared Docker-network service name or a reachable host address. |
+| Settings field is blank after reload | This is intentional. Leaving it blank preserves the saved destinations. |
+| Discord reports an invalid webhook | Verify both the webhook ID and token, or create a new Discord webhook. |
+| Messages repeat during an incident | Increase the failure cooldown and check whether the resource identifier is changing between events. |
+
+DockFlare logs include a secret-free event ID and aggregate delivery result. They do not include complete destination URLs.
+
+## Related documentation
+
+- [Discord Notifications and Apprise API](Discord-Apprise-Notifications.md)
+- [Apprise project](https://github.com/caronc/apprise)
+- [Apprise notification services](https://appriseit.com/services/)
+- [Apprise API project](https://github.com/caronc/apprise-api)
+- [Discord webhook documentation](https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks)

dockflare/app/templates/docs/en/Discord-Apprise-Notifications.md

@@ -0,0 +1,321 @@
+# Discord Notifications and Apprise API
+
+This guide configures a Discord channel to receive DockFlare notifications. It covers two supported designs:
+
+1. Paste the Discord webhook directly into DockFlare. Nothing else needs to be installed.
+2. Deploy the optional Apprise API, store Discord there, and connect DockFlare to that stored configuration.
+
+The first setup is recommended when DockFlare is the only application sending notifications. Apprise is already bundled inside the DockFlare container and recognizes the normal Discord webhook URL. The API setup is useful when several applications should share destinations or when notification configuration should be managed centrally.
+
+Apprise is an independent third-party open-source project. It is not developed or operated by DockFlare. Notifications remain disabled until you explicitly configure and enable them.
+
+## Before you begin
+
+You need:
+
+- permission to manage webhooks in the target Discord server;
+- a Discord text channel for DockFlare messages;
+- access to DockFlare's **Settings** page; and
+- Docker with Compose if you choose the self-hosted Apprise API setup.
+
+Never post a real webhook URL, configuration key, or Apprise destination in an issue, screenshot, chat, or log. The examples below use placeholders.
+
+## Step 1: Create a Discord webhook
+
+1. Open the Discord server.
+2. Open **Server Settings**.
+3. Select **Integrations**.
+4. Select **Webhooks**.
+5. Create a webhook.
+6. Name it `DockFlare` and select the channel that should receive notifications.
+7. Select **Copy Webhook URL**.
+
+The copied URL has this structure:
+
+```text
+https://discord.com/api/webhooks/{WEBHOOK_ID}/{WEBHOOK_TOKEN}
+```
+
+The ID and token authorize anyone holding the URL to post to the selected channel. Store it as a secret. If it is exposed, delete or regenerate the webhook in Discord.
+
+## Option A: Paste the Discord webhook into DockFlare
+
+This is the simplest setup. No Apprise installation or additional container is required.
+
+Paste the complete webhook copied from Discord into DockFlare without changing it:
+
+```text
+https://discord.com/api/webhooks/{WEBHOOK_ID}/{WEBHOOK_TOKEN}
+```
+
+For example:
+
+```text
+https://discord.com/api/webhooks/123456789012345678/REPLACE_WITH_THE_REAL_TOKEN
+```
+
+Apprise also supports its explicit Discord URL format, but converting the webhook is optional:
+
+```text
+discord://123456789012345678/REPLACE_WITH_THE_REAL_TOKEN
+```
+
+Then continue with [Configure DockFlare](#configure-dockflare).
+
+## Option B: Deploy a self-hosted Apprise API
+
+The Apprise API provides a web interface and persistent configuration store. DockFlare sends to the API, and the API forwards the notification to Discord.
+
+The data flow is:
+
+```text
+DockFlare -> Apprise API -> Discord webhook -> Discord channel
+```
+
+### Create the deployment directory
+
+Run these commands on the Docker host that will run Apprise:
+
+```bash
+mkdir -p apprise/config
+cd apprise
+```
+
+Create `docker-compose.yml` with the following content:
+
+```yaml
+services:
+  apprise:
+    image: caronc/apprise:latest
+    container_name: apprise
+    restart: unless-stopped
+    user: "${PUID:-1000}:${PGID:-1000}"
+    ports:
+      - "8000:8000"
+    environment:
+      APPRISE_STATEFUL_MODE: simple
+      APPRISE_WORKER_COUNT: "1"
+      APPRISE_ADMIN: "y"
+      TZ: ${TZ:-UTC}
+    volumes:
+      - ./config:/config
+    networks:
+      - cloudflare-net
+
+networks:
+  cloudflare-net:
+    external: true
+```
+
+Create a `.env` file beside `docker-compose.yml` and set the user, group, and timezone for your Docker host:
+
+```dotenv
+PUID=1000
+PGID=1000
+TZ=Europe/Zurich
+```
+
+Use `id -u` and `id -g` on the host to find the correct numeric values. The example uses DockFlare's existing external `cloudflare-net` network so the DockFlare container can reach the API by its service name.
+
+If your DockFlare deployment uses a different shared network, replace `cloudflare-net` with that network. If Apprise runs on another host, the shared Docker network is not required and DockFlare must use a reachable hostname or IP address instead.
+
+If the external network does not exist yet, create it before starting the stack:
+
+```bash
+docker network create cloudflare-net
+```
+
+### Start Apprise API
+
+```bash
+docker compose pull
+docker compose up -d
+```
+
+Open the web interface:
+
+```text
+http://{APPRISE_HOST}:8000
+```
+
+You can also verify the API status at:
+
+```text
+http://{APPRISE_HOST}:8000/status
+```
+
+This Compose example is intended for a trusted local network. Do not expose the administrative interface directly to the public internet. Use firewall restrictions or a properly authenticated HTTPS reverse proxy if remote administration is required.
+
+### Create a persistent configuration
+
+1. Open the Apprise API web interface.
+2. Open the configuration manager.
+3. Create a persistent configuration.
+4. Note the configuration key shown by Apprise. It appears in an address resembling `/cfg/{CONFIG_KEY}`.
+5. Add the Discord destination:
+
+```text
+discord://{WEBHOOK_ID}/{WEBHOOK_TOKEN}
+```
+
+6. Save the configuration.
+7. Use Apprise's test function and confirm exactly one message reaches the Discord channel.
+
+The configuration is stored below the local `./config` directory mounted at `/config` in the container. Include this directory in your normal backup process and protect it because it contains notification secrets.
+
+### Build the URL for DockFlare
+
+The address visible in your browser is not the destination to paste into DockFlare.
+
+Do not use:
+
+```text
+http://{APPRISE_HOST}:8000/cfg/{CONFIG_KEY}#overview
+```
+
+Use the Apprise API plugin URL shown on the configuration overview. Its format is:
+
+```text
+apprise://{APPRISE_HOST}:8000/{CONFIG_KEY}
+```
+
+Choose the host according to your deployment:
+
+| Deployment | DockFlare destination example |
+|---|---|
+| Apprise and DockFlare share a Docker network | `apprise://apprise:8000/{CONFIG_KEY}` |
+| Apprise runs on another reachable host | `apprise://192.0.2.10:8000/{CONFIG_KEY}` |
+| Apprise is behind an HTTPS reverse proxy | `apprises://notify.example.com/{CONFIG_KEY}` |
+
+`apprise://` uses HTTP. `apprises://` uses HTTPS.
+
+Do not use `localhost` or `127.0.0.1` when Apprise is a separate container. From inside the DockFlare container, those addresses refer to DockFlare itself.
+
+Treat the complete API plugin URL as a secret because its configuration key can trigger every destination stored under that key.
+
+## Configure DockFlare
+
+1. Open DockFlare.
+2. Go to **Settings > Notifications**.
+3. Paste one destination into **Replacement Apprise URLs**:
+   - Option A: `https://discord.com/api/webhooks/{WEBHOOK_ID}/{WEBHOOK_TOKEN}`
+   - Option B: `apprise://apprise:8000/{CONFIG_KEY}` or the equivalent URL for your deployment
+4. Select the notification events you want to receive.
+5. Enable notifications.
+6. Select **Save notification settings**.
+7. Select **Send test notification**.
+8. Wait for DockFlare to report successful delivery.
+9. Confirm exactly one test message appears in Discord.
+
+The replacement field is empty after every page load. This is intentional. Leaving it empty preserves the saved destination. Entering a value replaces the complete destination list.
+
+## Recommended event selection
+
+A useful starting point is:
+
+- Cloudflare tunnel, DNS, and Access failures
+- Docker listener stopped
+- Agent enrollment failed
+- Agent offline and recovered
+- Agent decommission completed, failed, and timed out
+- Tunnel down and recovered
+- Rule deleted
+
+Enable rule activation, pending deletion, decommission start, and Access Policy management events if you want a more detailed administrative activity stream.
+
+Access Policy lifecycle messages confirm that a policy definition changed. They do not mean the policy has been assigned to a service.
+
+## Verify a real event
+
+After the test message succeeds, verify one disposable lifecycle event:
+
+1. Enable **Rule activated**.
+2. Start a disposable DockFlare-enabled container or create a disposable manual rule.
+3. Wait for its tunnel, DNS, and optional Access work to complete.
+4. Confirm Discord receives one activation message with a clickable HTTPS service URL.
+5. Remove the disposable resource and confirm DockFlare returns to its previous state.
+
+Do not use a production service solely for notification testing.
+
+## Troubleshooting
+
+### DockFlare rejects the destination
+
+- For a direct Discord destination, confirm you pasted the complete `https://discord.com/api/webhooks/{WEBHOOK_ID}/{WEBHOOK_TOKEN}` URL. The optional `discord://` format is also accepted.
+- For an Apprise API destination, confirm the scheme is `apprise://` or `apprises://`.
+- Confirm the URL contains no placeholder braces.
+- Confirm the webhook ID, token, API host, port, and configuration key are complete.
+
+### Apprise works in the browser but DockFlare cannot reach it
+
+From the DockFlare container, verify that the API hostname resolves and port 8000 is reachable. The browser reaching a host address does not prove that the DockFlare container can reach the same address.
+
+For a shared network, confirm both containers are attached:
+
+```bash
+docker network inspect cloudflare-net
+```
+
+The container name `apprise` must appear in the network membership before this URL can work:
+
+```text
+apprise://apprise:8000/{CONFIG_KEY}
+```
+
+### The API returns configuration-not-found or HTTP 404
+
+- Copy the configuration key again from Apprise.
+- Confirm the persistent configuration was saved.
+- Confirm `./config` is mounted at `/config`.
+- Confirm DockFlare is pointing to the correct Apprise API instance.
+
+### Discord receives no message
+
+Test each link in order:
+
+1. Test the Discord destination from Apprise, or test the direct destination in DockFlare.
+2. Test the stored Apprise configuration from the Apprise web interface.
+3. Test delivery from DockFlare.
+4. Trigger a disposable real event.
+
+The first failing step identifies which connection needs attention.
+
+### Discord receives duplicate messages
+
+- Confirm the same Discord webhook is not listed more than once in the stored Apprise configuration.
+- Confirm DockFlare does not contain both the direct Discord URL and an Apprise API URL that forwards to the same webhook.
+- Check whether separate DockFlare instances are using the same destination.
+
+### A saved DockFlare destination is not visible
+
+DockFlare never displays complete saved destination URLs. A blank replacement field does not mean the destination was removed. Check the redacted destination count and scheme instead.
+
+## Maintain the Apprise API
+
+Back up the deployment's `config` directory before upgrading.
+
+Update the container with:
+
+```bash
+docker compose pull
+docker compose up -d
+```
+
+Verify `/status`, send an Apprise test, and then send a DockFlare test after the upgrade.
+
+## Security checklist
+
+- Keep Discord webhook URLs and Apprise configuration keys secret.
+- Restrict access to port 8000 and the Apprise administrative interface.
+- Use HTTPS when traffic crosses an untrusted network.
+- Protect the persistent `config` directory and its backups.
+- Rotate a Discord webhook immediately if its URL is exposed.
+- Configure only notification destinations you trust.
+- Never include real secrets in screenshots, support requests, or logs.
+
+## Related documentation
+
+- [Notifications with Apprise](Apprise-Notifications.md)
+- [Apprise API project](https://github.com/caronc/apprise-api)
+- [Apprise Discord service](https://appriseit.com/services/discord/)
+- [Discord webhooks](https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks)

dockflare/app/templates/docs/en/navigation.md

@@ -20,6 +20,8 @@ This documentation provides comprehensive information for DockFlare. Whether you
     *   [State-Persistence](State-Persistence.md)
 *   **Configuration**
     *   [Container Labels](Container-Labels.md)
+    *   [Apprise Notifications](Apprise-Notifications.md)
+    *   [Discord & Apprise API Setup](Discord-Apprise-Notifications.md)
     *   [Identity Providers](Identity-Providers.md)
     *   [OAuth Provider Setup](OAuth-Provider-Setup.md)
 *   **Usage Guide**

dockflare/app/templates/docs/es/navigation.md

@@ -20,6 +20,8 @@ Esta documentación proporciona información completa sobre DockFlare. Ya seas u
     * [Persistencia del estado](State-Persistence.md)
 * **Configuración**
     * [Etiquetas de contenedor](Container-Labels.md)
+    * [Apprise Notifications (English)](Apprise-Notifications.md)
+    * [Discord & Apprise API Setup (English)](Discord-Apprise-Notifications.md)
     * [Proveedores de identidad](Identity-Providers.md)
     * [Configuración del proveedor OAuth](OAuth-Provider-Setup.md)
 * **Guía de uso**

dockflare/app/templates/docs/fr/navigation.md

@@ -20,6 +20,8 @@ Cette documentation couvre DockFlare de bout en bout. Que vous démarriez ou que
     * [Persistance de l'état](State-Persistence.md)
 * **Configuration**
     * [Étiquettes des conteneurs](Container-Labels.md)
+    * [Apprise Notifications (English)](Apprise-Notifications.md)
+    * [Discord & Apprise API Setup (English)](Discord-Apprise-Notifications.md)
     * [Fournisseurs d'identité](Identity-Providers.md)
     * [Configuration du fournisseur OAuth](OAuth-Provider-Setup.md)
 * **Guide d'utilisation**

dockflare/app/templates/docs/id/navigation.md

@@ -20,6 +20,8 @@ Dokumentasi ini menyajikan panduan lengkap untuk DockFlare. Baik Anda pengguna b
     *   [Persistensi State](State-Persistence.md)
 *   **Konfigurasi**
     *   [Label Container](Container-Labels.md)
+    *   [Apprise Notifications (English)](Apprise-Notifications.md)
+    *   [Discord & Apprise API Setup (English)](Discord-Apprise-Notifications.md)
     *   [Identity Provider](Identity-Providers.md)
     *   [Setup OAuth Provider](OAuth-Provider-Setup.md)
 *   **Panduan Penggunaan**

dockflare/app/templates/docs/it/navigation.md

@@ -20,6 +20,8 @@ Questa documentazione fornisce informazioni complete su DockFlare. Che tu sia un
     * [Persistenza dello stato](State-Persistence.md)
 * **Configurazione**
     * [Etichette dei container](Container-Labels.md)
+    * [Apprise Notifications (English)](Apprise-Notifications.md)
+    * [Discord & Apprise API Setup (English)](Discord-Apprise-Notifications.md)
     * [Fornitori di identità](Identity-Providers.md)
     * [Configurazione provider OAuth](OAuth-Provider-Setup.md)
 * **Guida all'uso**

dockflare/app/templates/docs/ja/navigation.md

@@ -20,6 +20,8 @@ DockFlare は、Cloudflare Tunnel と Zero Trust の管理を簡素化する自
     * [状態永続性](State-Persistence.md)
 * **構成**
     * [コンテナラベル](Container-Labels.md)
+    * [Apprise Notifications (English)](Apprise-Notifications.md)
+    * [Discord & Apprise API Setup (English)](Discord-Apprise-Notifications.md)
     * [アイデンティティプロバイダー](Identity-Providers.md)
     * [OAuth プロバイダーのセットアップ](OAuth-Provider-Setup.md)
 * **使用ガイド**

dockflare/app/templates/docs/pl/navigation.md

@@ -20,6 +20,8 @@ Niniejsza dokumentacja zawiera wyczerpujące informacje na temat DockFlare. Niez
     * [Trwałość stanu](State-Persistence.md)
 * **Konfiguracja**
     * [Etykiety kontenerów](Container-Labels.md)
+    * [Apprise Notifications (English)](Apprise-Notifications.md)
+    * [Discord & Apprise API Setup (English)](Discord-Apprise-Notifications.md)
     * [Dostawcy tożsamości](Identity-Providers.md)
     * [Konfiguracja dostawcy OAuth](OAuth-Provider-Setup.md)
 * **Przewodnik użytkowania**

dockflare/app/templates/docs/sk/navigation.md

@@ -20,6 +20,8 @@ Táto dokumentácia poskytuje komplexné informácie o DockFlare. Či už si nov
     *   [Trvalé uloženie stavu](State-Persistence.md)
 *   **Konfigurácia**
     *   [Docker labely](Container-Labels.md)
+    *   [Apprise Notifications (English)](Apprise-Notifications.md)
+    *   [Discord & Apprise API Setup (English)](Discord-Apprise-Notifications.md)
     *   [Poskytovatelia identity](Identity-Providers.md)
     *   [Nastavenie poskytovateľa OAuth](OAuth-Provider-Setup.md)
 *   **Návod na použitie**

dockflare/app/templates/docs/zh/navigation.md

@@ -20,6 +20,8 @@ DockFlare 是一款功能强大的自托管入口控制器，可简化 Cloudflar
     * [状态持久化](State-Persistence.md)
 * **配置**
     * [容器标签](Container-Labels.md)
+    * [Apprise Notifications (English)](Apprise-Notifications.md)
+    * [Discord & Apprise API Setup (English)](Discord-Apprise-Notifications.md)
     * [身份提供商](Identity-Providers.md)
     * [OAuth 提供程序设置](OAuth-Provider-Setup.md)
 * **使用指南**

dockflare/app/templates/settings.html

@@ -18,6 +18,7 @@
                 <li><a href="{{ url_for('web.settings_page') }}#backup-restore">{{ t('settings.backup_restore') }}</a></li>
                 <li><a href="{{ url_for('web.settings_page') }}#security">{{ t('settings.security') }}</a></li>
                 <li><a href="{{ url_for('web.settings_page') }}#security" class="pl-8 text-sm">{{ t('settings.oauth_authentication') }}</a></li>
+                <li><a href="{{ url_for('web.settings_page') }}#notifications">{{ t('settings.notifications') }}</a></li>
                 <li><a href="{{ url_for('web.settings_page') }}#tunnel-agent-status">{{ t('settings.tunnel_agent_status') }}</a></li>
             </ul>
             <div class="mt-4 px-2">
@@ -452,7 +453,124 @@
             </div>
         </section>
 
-        <!-- 5. Tunnel & Agent Status Section -->
+        <section id="notifications" class="card bg-base-100 shadow-xl mb-8 sm:mb-12 transition-all duration-300 hover:shadow-2xl">
+            <div class="card-body">
+                <h2 class="card-title text-2xl sm:text-3xl border-b border-base-300 pb-3 mb-6">
+                    {{ t('settings.notifications') }}
+                </h2>
+                <p class="text-sm opacity-70">{{ t('settings.notifications_description') }}</p>
+
+                <div class="rounded-box bg-base-200 p-4 mt-4 text-sm space-y-2">
+                    <p><strong>{{ t('settings.notification_status') }}:</strong>
+                        {% if notification_status.enabled and notification_status.available %}
+                            <span class="badge badge-success">{{ t('common.active') }}</span>
+                        {% elif notification_status.enabled %}
+                            <span class="badge badge-error">{{ t('settings.notification_unavailable') }}</span>
+                        {% else %}
+                            <span class="badge badge-ghost">{{ t('settings.notification_disabled') }}</span>
+                        {% endif %}
+                    </p>
+                    <p><strong>{{ t('settings.configured_destinations') }}:</strong> {{ notification_status.configured_destination_count }}</p>
+                    {% if notification_status.destinations %}
+                        <ul class="list-disc list-inside opacity-80">
+                            {% for destination in notification_status.destinations %}
+                                <li>{{ destination.summary }}</li>
+                            {% endfor %}
+                        </ul>
+                    {% endif %}
+                </div>
+
+                <form method="POST" action="{{ url_for('web.save_notification_settings') }}" class="space-y-5 mt-5">
+                    {{ notification_settings_form.hidden_tag() }}
+                    <div class="form-control">
+                        <label class="label cursor-pointer justify-start gap-3">
+                            {{ notification_settings_form.enabled(class="toggle toggle-primary") }}
+                            <span class="label-text">{{ t('settings.enable_notifications') }}</span>
+                        </label>
+                    </div>
+                    <div class="form-control">
+                        <label class="label" for="notification-replacement-urls">
+                            <span class="label-text">{{ t('settings.replacement_apprise_urls') }}</span>
+                        </label>
+                        {{ notification_settings_form.replacement_urls(class="textarea textarea-bordered min-h-32 font-mono text-sm", id="notification-replacement-urls", placeholder="discord://...\ntgram://...") }}
+                        <label class="label"><span class="label-text-alt">{{ t('settings.replacement_apprise_urls_help') }}</span></label>
+                    </div>
+                    <div class="form-control">
+                        <label class="label cursor-pointer justify-start gap-3">
+                            {{ notification_settings_form.clear_urls(class="checkbox checkbox-error") }}
+                            <span class="label-text">{{ t('settings.clear_notification_destinations') }}</span>
+                        </label>
+                    </div>
+                    <div class="form-control max-w-xs">
+                        <label class="label" for="notification-cooldown"><span class="label-text">{{ t('settings.notification_cooldown') }}</span></label>
+                        {{ notification_settings_form.failure_cooldown_seconds(class="input input-bordered", id="notification-cooldown", min="60", max="86400") }}
+                        <label class="label"><span class="label-text-alt">{{ t('settings.notification_cooldown_help') }}</span></label>
+                    </div>
+
+                    {% set notification_event_groups = [
+                        ('settings.notification_group_rules', [
+                            ('rule_activated', 'settings.notification_event_rule_activated'),
+                            ('rule_restored', 'settings.notification_event_rule_restored'),
+                            ('rule_pending_deletion', 'settings.notification_event_rule_pending_deletion'),
+                            ('rule_deleted', 'settings.notification_event_rule_deleted')
+                        ]),
+                        ('settings.notification_group_failures', [
+                            ('cloudflare_tunnel_failure', 'settings.notification_event_tunnel_failure'),
+                            ('cloudflare_dns_failure', 'settings.notification_event_dns_failure'),
+                            ('cloudflare_access_failure', 'settings.notification_event_access_failure'),
+                            ('docker_listener_failure', 'settings.notification_event_docker_listener_failure')
+                        ]),
+                        ('settings.notification_group_agents', [
+                            ('agent_enrolled', 'settings.notification_event_agent_enrolled'),
+                            ('agent_enrollment_failed', 'settings.notification_event_agent_enrollment_failed'),
+                            ('agent_offline', 'settings.notification_event_agent_offline'),
+                            ('agent_online', 'settings.notification_event_agent_online'),
+                            ('agent_decommission_started', 'settings.notification_event_agent_decommission_started'),
+                            ('agent_decommission_completed', 'settings.notification_event_agent_decommission_completed'),
+                            ('agent_decommission_failed', 'settings.notification_event_agent_decommission_failed'),
+                            ('agent_decommission_stalled', 'settings.notification_event_agent_decommission_stalled')
+                        ]),
+                        ('settings.notification_group_tunnels', [
+                            ('tunnel_down', 'settings.notification_event_tunnel_down'),
+                            ('tunnel_recovered', 'settings.notification_event_tunnel_recovered')
+                        ]),
+                        ('settings.notification_group_access_policies', [
+                            ('access_policy_created', 'settings.notification_event_access_policy_created'),
+                            ('access_policy_updated', 'settings.notification_event_access_policy_updated'),
+                            ('access_policy_deleted', 'settings.notification_event_access_policy_deleted')
+                        ])
+                    ] %}
+                    <fieldset class="space-y-4">
+                        <legend class="font-semibold mb-3">{{ t('settings.notification_events') }}</legend>
+                        {% for group_label_key, notification_events in notification_event_groups %}
+                            <div>
+                                <h4 class="text-sm font-semibold mb-2">{{ t(group_label_key) }}</h4>
+                                <div class="grid grid-cols-1 md:grid-cols-2 gap-3">
+                                    {% for field_name, label_key in notification_events %}
+                                        <label class="label cursor-pointer justify-start gap-3 rounded-md bg-base-200/50 px-3">
+                                            {{ notification_settings_form[field_name](class="checkbox checkbox-sm checkbox-primary") }}
+                                            <span class="label-text">{{ t(label_key) }}</span>
+                                        </label>
+                                    {% endfor %}
+                                </div>
+                            </div>
+                        {% endfor %}
+                    </fieldset>
+
+                    <div class="card-actions pt-2 gap-3">
+                        {{ notification_settings_form.submit_notifications(class="btn btn-primary", value=t('settings.save_notification_settings')) }}
+                        <button type="button" id="btn-test-notifications" class="btn btn-outline" {% if not notification_status.enabled or not notification_status.available %}disabled{% endif %}>{{ t('settings.send_test_notification') }}</button>
+                    </div>
+                    <div id="notification-test-status" class="text-sm hidden" role="status" aria-live="polite"></div>
+                </form>
+
+                <div role="alert" class="alert alert-warning text-sm mt-5">
+                    <span>{{ t('settings.notification_security_warning') }}</span>
+                </div>
+            </div>
+        </section>
+
+        <!-- 6. Tunnel & Agent Status Section -->
         <section id="tunnel-agent-status" class="card bg-base-100 shadow-xl mb-8 sm:mb-12 transition-all duration-300 hover:shadow-2xl">
             <div class="card-body">
                 <h2 class="card-title text-2xl sm:text-3xl border-b border-base-300 pb-3 mb-6">
@@ -536,6 +654,55 @@
 <script>
   (function () {
 
+    const testNotificationsBtn = document.getElementById('btn-test-notifications');
+    const notificationTestStatus = document.getElementById('notification-test-status');
+    if (testNotificationsBtn && notificationTestStatus) {
+      const setNotificationTestStatus = (message, isError) => {
+        notificationTestStatus.textContent = message;
+        notificationTestStatus.classList.remove('hidden', 'text-error', 'text-success');
+        notificationTestStatus.classList.add(isError ? 'text-error' : 'text-success');
+      };
+      const pollNotificationTest = async (jobId) => {
+        for (let attempt = 0; attempt < 30; attempt += 1) {
+          await new Promise(resolve => setTimeout(resolve, 1000));
+          const response = await fetch(`/api/v2/notifications/test/${encodeURIComponent(jobId)}`, {credentials: 'same-origin'});
+          const payload = await response.json().catch(() => ({}));
+          if (payload.status === 'success') {
+            setNotificationTestStatus("{{ t('settings.test_notification_success') }}", false);
+            return;
+          }
+          if (!response.ok || payload.status === 'failure' || payload.status === 'not_found') {
+            setNotificationTestStatus("{{ t('settings.test_notification_failed') }}", true);
+            return;
+          }
+        }
+        setNotificationTestStatus("{{ t('settings.test_notification_timeout') }}", true);
+      };
+      testNotificationsBtn.addEventListener('click', async () => {
+        testNotificationsBtn.disabled = true;
+        notificationTestStatus.textContent = "{{ t('settings.test_notification_pending') }}";
+        notificationTestStatus.classList.remove('hidden', 'text-error', 'text-success');
+        try {
+          const response = await fetch('/api/v2/notifications/test', {
+            method: 'POST',
+            headers: {'Content-Type': 'application/json', 'X-CSRFToken': '{{ csrf_token() }}'},
+            credentials: 'same-origin',
+            body: '{}'
+          });
+          const payload = await response.json().catch(() => ({}));
+          if (!response.ok || !payload.job_id) {
+            setNotificationTestStatus("{{ t('settings.test_notification_failed') }}", true);
+            return;
+          }
+          await pollNotificationTest(payload.job_id);
+        } catch (error) {
+          setNotificationTestStatus("{{ t('settings.test_notification_failed') }}", true);
+        } finally {
+          testNotificationsBtn.disabled = false;
+        }
+      });
+    }
+
     const showMasterKeyBtn = document.getElementById('btn-show-master-key');
     if (showMasterKeyBtn) {
       showMasterKeyBtn.addEventListener('click', async () => {

dockflare/app/templates/status_page.html

@@ -248,9 +248,12 @@
                                         {% if details.access_policy_ui_override and details.source != 'manual' %}
                                             <span class="badge badge-warning badge-xs ml-2 animate-pulse" title="{{ t('status.ui_policy_override') }}">{{ t('status.ui_override') }}</span>
                                         {% endif %}
-                                        {% if details.rule_ui_override and details.source == 'docker' %}
+                                        {% if details.rule_ui_override and details.source in ['docker', 'agent'] %}
                                             <span class="badge badge-error badge-xs ml-2 animate-pulse" title="{{ t('status.docker_rule_overridden') }}">{{ t('status.rule_ui_override') }}</span>
                                         {% endif %}
+                                        {% if details.tunnel_sync_pending %}
+                                            <span class="badge badge-warning badge-xs ml-2 animate-pulse" title="{{ t('status.tunnel_sync_pending_help') }}">{{ t('status.tunnel_sync_pending') }}</span>
+                                        {% endif %}
                                     </td>
                                     <td class="p-3 whitespace-nowrap text-sm opacity-70" data-role="expires-cell" data-label="{{ t('status.expires_in') }}">
                                         {% if details.status=='pending_deletion' and details.delete_at %}
@@ -312,7 +315,7 @@
                                                     </a>
                                                 </li>
                                                 {% endif %}
-                                                {% if details.rule_ui_override and details.source == 'docker' %}
+                                                {% if details.rule_ui_override and details.source in ['docker', 'agent'] %}
                                                 <li>
                                                     <a class="revert-rule-btn" data-rule-key="{{ hostname }}" data-display-hostname="{{ display_hostname }}">
                                                         <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-4 h-4">

dockflare/app/web/config_loader.py

@@ -20,12 +20,14 @@
 import json
 import logging
 import os
+import tempfile
 from typing import Dict, Optional
 
 from cryptography.fernet import Fernet
 
 from app import config
 from app.core.container_name import build_cloudflared_container_name
+from app.core.notification_manager import normalize_notification_config
 
 
 def _data_directory() -> str:
@@ -67,6 +69,36 @@ def load_encrypted_config() -> Optional[Dict]:
     return data
 
 
+def save_encrypted_config(config_data: Dict, fernet: Fernet) -> bool:
+    if not isinstance(config_data, dict) or fernet is None:
+        return False
+    destination = config_file_path()
+    temporary_path = None
+    try:
+        payload = fernet.encrypt(json.dumps(config_data).encode("utf-8"))
+        os.makedirs(os.path.dirname(destination), exist_ok=True)
+        with tempfile.NamedTemporaryFile(
+            mode="wb",
+            dir=os.path.dirname(destination),
+            prefix=".dockflare_config.",
+            delete=False,
+        ) as temporary:
+            temporary.write(payload)
+            temporary.flush()
+            os.fsync(temporary.fileno())
+            temporary_path = temporary.name
+        os.replace(temporary_path, destination)
+        return True
+    except Exception as err:
+        logging.error("CONFIG_LOADER: Failed to save encrypted config: %s", type(err).__name__)
+        if temporary_path:
+            try:
+                os.unlink(temporary_path)
+            except OSError:
+                pass
+        return False
+
+
 def apply_config_to_app(flask_app, config_data: Dict) -> None:
     """Populate Flask app and module-level config values from decrypted data."""
     if not isinstance(config_data, dict):
@@ -134,6 +166,10 @@ def apply_config_to_app(flask_app, config_data: Dict) -> None:
     config.EMAIL_ENABLED = flask_app.config['EMAIL_ENABLED']
     config.EMAIL_CONFIG = flask_app.config['EMAIL_CONFIG']
 
+    notification_config = normalize_notification_config(config_data.get('notification_config'))
+    flask_app.config['NOTIFICATION_CONFIG'] = notification_config
+    config.NOTIFICATION_CONFIG = notification_config
+
     if flask_app.config['CF_API_TOKEN']:
         config.CF_HEADERS['Authorization'] = f"Bearer {flask_app.config['CF_API_TOKEN']}"
     else:

dockflare/app/web/forms.py

@@ -16,8 +16,8 @@
 #
 # dockflare/app/web/forms.py
 from flask_wtf import FlaskForm
-from wtforms import BooleanField, PasswordField, SubmitField, StringField, IntegerField
-from wtforms.validators import DataRequired, EqualTo, Length, Optional
+from wtforms import BooleanField, PasswordField, SubmitField, StringField, IntegerField, TextAreaField
+from wtforms.validators import DataRequired, EqualTo, Length, NumberRange, Optional
 
 class SettingsForm(FlaskForm):
     """Form for editing general application settings."""
@@ -109,3 +109,36 @@ class CloudflareCredentialsForm(FlaskForm):
         validators=[Optional(), Length(min=40, max=100, message="API Token must be at least 40 characters long.")]
     )
     submit_cloudflare_credentials = SubmitField('Update Cloudflare Credentials')
+
+
+class NotificationSettingsForm(FlaskForm):
+    enabled = BooleanField('Enable notifications')
+    replacement_urls = TextAreaField('Replacement Apprise URLs', validators=[Optional(), Length(max=131072)])
+    clear_urls = BooleanField('Clear configured destinations')
+    failure_cooldown_seconds = IntegerField(
+        'Failure cooldown (seconds)',
+        validators=[DataRequired(), NumberRange(min=60, max=86400)],
+        default=900,
+    )
+    rule_activated = BooleanField('Rule activated')
+    rule_restored = BooleanField('Rule restored')
+    rule_pending_deletion = BooleanField('Rule pending deletion')
+    rule_deleted = BooleanField('Rule deleted')
+    cloudflare_tunnel_failure = BooleanField('Cloudflare tunnel failure')
+    cloudflare_dns_failure = BooleanField('Cloudflare DNS failure')
+    cloudflare_access_failure = BooleanField('Cloudflare Access failure')
+    docker_listener_failure = BooleanField('Docker listener failure')
+    agent_offline = BooleanField('Agent offline')
+    agent_online = BooleanField('Agent recovered')
+    agent_enrolled = BooleanField('Agent enrolled')
+    agent_enrollment_failed = BooleanField('Agent enrollment failed')
+    agent_decommission_started = BooleanField('Agent decommission started')
+    agent_decommission_completed = BooleanField('Agent decommission completed')
+    agent_decommission_failed = BooleanField('Agent decommission failed')
+    agent_decommission_stalled = BooleanField('Agent decommission stalled')
+    tunnel_down = BooleanField('Tunnel down')
+    tunnel_recovered = BooleanField('Tunnel recovered')
+    access_policy_created = BooleanField('Access Policy created')
+    access_policy_updated = BooleanField('Access Policy updated')
+    access_policy_deleted = BooleanField('Access Policy deleted')
+    submit_notifications = SubmitField('Save notification settings')

dockflare/app/web/routes.py

@@ -36,7 +36,15 @@ from app.core.user import User
 
 from app import config, docker_client, tunnel_state, cloudflared_agent_state, log_queue, state_update_queue, publish_state_event, limiter
 from app.core.cache import CACHE_ENABLED
-from app.core.state_manager import managed_rules, access_groups, state_lock, save_state, load_state
+from app.core.state_manager import (
+    access_groups,
+    agent_inventory_contains_rule,
+    get_agent,
+    load_state,
+    managed_rules,
+    save_state,
+    state_lock,
+)
 from app.core.tunnel_manager import (
     start_cloudflared_container,
     stop_cloudflared_container,
@@ -79,6 +87,7 @@ from app.core.docker_handler import is_valid_hostname, is_valid_service
 from app.core.utils import get_rule_key, normalize_path_value
 from app.core.container_name import build_cloudflared_container_name
 from app.core import backup_manager
+from app.core import notification_manager
 from app.web import config_loader
 from app.i18n import t as _t
 from cryptography.fernet import Fernet
@@ -457,7 +466,13 @@ def status_page():
                         default_tunnel_id=default_tunnel_id_value
                         )
 
-from app.web.forms import ChangePasswordForm, SecuritySettingsForm, SettingsForm, CloudflareCredentialsForm
+from app.web.forms import (
+    ChangePasswordForm,
+    CloudflareCredentialsForm,
+    NotificationSettingsForm,
+    SecuritySettingsForm,
+    SettingsForm,
+)
 from werkzeug.security import check_password_hash, generate_password_hash
 from cryptography.fernet import Fernet
 
@@ -548,6 +563,7 @@ def settings_page():
     change_password_form = ChangePasswordForm()
     security_settings_form = SecuritySettingsForm(prefix='security')
     cf_credentials_form = CloudflareCredentialsForm(prefix='cf_creds')
+    notification_settings_form = NotificationSettingsForm(prefix='notifications')
 
     
     if request.method == 'POST':
@@ -604,6 +620,7 @@ def settings_page():
                 if tunnel_name_changed and not config.USE_EXTERNAL_CLOUDFLARED:
                     flash(_t('flash.tunnel_name_changed'), 'info')
                     logging.info(f"Tunnel name changed from '{original_tunnel_name}' to '{new_tunnel_name}'. Triggering agent restart.")
+                    notification_manager.suppress_tunnel_health(tunnel_state.get("id"), 300)
                     
                     def restart_agent_task():
                         stop_cloudflared_container()
@@ -705,6 +722,13 @@ def settings_page():
         settings_form.preserve_unmanaged_cf_ingress_fields.data = current_app.config.get('PRESERVE_UNMANAGED_CF_INGRESS_FIELDS', False)
         settings_form.dockflare_public_url.data = current_app.config.get('DOCKFLARE_PUBLIC_URL', '')
         security_settings_form.disable_password_login.data = current_app.config.get('DISABLE_PASSWORD_LOGIN', False)
+        notification_config = current_app.config.get('NOTIFICATION_CONFIG', {})
+        notification_settings_form.enabled.data = bool(notification_config.get('enabled', False))
+        notification_settings_form.failure_cooldown_seconds.data = notification_config.get('failure_cooldown_seconds', 900)
+        for event_key, enabled in notification_config.get('events', {}).items():
+            field = getattr(notification_settings_form, event_key, None)
+            if field is not None:
+                field.data = bool(enabled)
 
     template_tunnel_state = {}
     template_agent_state = {}
@@ -723,6 +747,8 @@ def settings_page():
         change_password_form=change_password_form,
         security_settings_form=security_settings_form,
         cf_credentials_form=cf_credentials_form,
+        notification_settings_form=notification_settings_form,
+        notification_status=notification_manager.get_public_status(),
         all_account_tunnels=all_account_tunnels_list,
         tunnel_state=template_tunnel_state,
         agent_state=template_agent_state,
@@ -735,6 +761,102 @@ def settings_page():
         ACCOUNT_ID_FOR_DISPLAY=cf_account_id if cf_account_id else "Not Configured"
     )
 
+
+@bp.route('/settings/notifications', methods=['POST'])
+@login_required
+@limiter.limit("10 per minute")
+def save_notification_settings():
+    form = NotificationSettingsForm(prefix='notifications')
+    if not form.validate_on_submit():
+        flash(_t('flash.notification_settings_invalid'), 'danger')
+        return redirect(url_for('web.settings_page') + '#notifications')
+
+    config_data, fernet = config_loader.load_encrypted_config_with_cipher()
+    if config_data is None or fernet is None:
+        flash(_t('flash.notification_settings_save_failed'), 'danger')
+        return redirect(url_for('web.settings_page') + '#notifications')
+
+    from app.core.notification_manager import normalize_notification_config
+
+    previous_raw = config_data.get('notification_config')
+    previous_raw = copy.deepcopy(previous_raw) if isinstance(previous_raw, dict) else {}
+    previous = normalize_notification_config(previous_raw)
+    replacement_lines = [line.strip() for line in (form.replacement_urls.data or '').splitlines() if line.strip()]
+
+    if form.clear_urls.data:
+        urls = []
+    elif replacement_lines:
+        valid, invalid = notification_manager.validate_urls(replacement_lines)
+        if not valid:
+            line_number, scheme = invalid[0]
+            safe_detail = f"line {line_number}" if line_number else "destination list"
+            if scheme not in {'unknown', 'invalid', 'unavailable'}:
+                safe_detail += f" ({scheme})"
+            flash(_t('flash.notification_destination_invalid', detail=safe_detail), 'danger')
+            return redirect(url_for('web.settings_page') + '#notifications')
+        urls = normalize_notification_config({'urls': replacement_lines})['urls']
+    else:
+        urls = previous['urls']
+
+    if form.enabled.data and not urls:
+        flash(_t('flash.notification_destination_required'), 'danger')
+        return redirect(url_for('web.settings_page') + '#notifications')
+
+    stored_events = previous_raw.get('events')
+    stored_events = copy.deepcopy(stored_events) if isinstance(stored_events, dict) else {}
+    for event_key in previous['events']:
+        stored_events[event_key] = bool(getattr(form, event_key).data)
+
+    updated_raw = previous_raw
+    updated_raw.update({
+        'enabled': bool(form.enabled.data),
+        'urls': urls,
+        'events': stored_events,
+        'failure_cooldown_seconds': int(form.failure_cooldown_seconds.data),
+    })
+    config_data['notification_config'] = updated_raw
+    if not config_loader.save_encrypted_config(config_data, fernet):
+        flash(_t('flash.notification_settings_save_failed'), 'danger')
+        return redirect(url_for('web.settings_page') + '#notifications')
+
+    config_loader.apply_config_to_app(current_app, config_data)
+    notification_manager.configure(current_app.config['NOTIFICATION_CONFIG'])
+    flash(_t('flash.notification_settings_saved'), 'success')
+    return redirect(url_for('web.settings_page') + '#notifications')
+
+
+@bp.route('/api/v2/notifications/test', methods=['POST'])
+@login_required
+@limiter.limit("5 per minute")
+def test_notification_delivery():
+    status = notification_manager.get_public_status()
+    if not status['enabled'] or not status['available']:
+        return jsonify({'status': 'error', 'message': 'notifications_not_available'}), 409
+    job_id, accepted = notification_manager.send_test()
+    if not accepted:
+        return jsonify({'status': 'error', 'message': 'notification_queue_unavailable'}), 503
+    return jsonify({'status': 'accepted', 'job_id': job_id}), 202
+
+
+@bp.route('/api/v2/notifications/test/<job_id>', methods=['GET'])
+@login_required
+def notification_test_status(job_id):
+    job = notification_manager.get_test_status(job_id)
+    if not job:
+        return jsonify({'status': 'not_found'}), 404
+    response = {'status': job['status']}
+    if job.get('completed_at'):
+        response['completed_at'] = job['completed_at']
+    if job['status'] == 'failure':
+        response['message'] = 'Notification delivery failed. Check DockFlare logs.'
+    return jsonify(response), 200
+
+
+@bp.route('/api/v2/notifications/status', methods=['GET'])
+@login_required
+def notification_public_status():
+    return jsonify(notification_manager.get_public_status()), 200
+
 @bp.route('/settings/reveal-master-key', methods=['POST'])
 @login_required
 def reveal_master_key():
@@ -842,9 +964,6 @@ def change_password():
 @bp.route('/revert_access_policy_to_labels/<path:hostname>', methods=['POST'])
 def revert_access_policy_to_labels(hostname):
     fqdn = hostname.split('|')[0]
-    if not docker_client:
-        return redirect(url_for('web.status_page'))
-    
     action_status_message = f"Attempting to revert Access Policy for '{fqdn}' to label configuration..."
     app_id_to_delete_if_any = None
     state_changed_for_revert = False
@@ -857,7 +976,10 @@ def revert_access_policy_to_labels(hostname):
             return redirect(url_for('web.status_page'))
         
         app_id_to_delete_if_any = current_rule.get("access_app_id")
+        source = current_rule.get("source")
+        agent_id = current_rule.get("agent_id")
         current_rule["access_policy_ui_override"] = False
+        current_rule["lifecycle_generation"] = int(current_rule.get("lifecycle_generation") or 0) + 1
         state_changed_for_revert = True
         if state_changed_for_revert:
             save_state()
@@ -867,8 +989,18 @@ def revert_access_policy_to_labels(hostname):
         if delete_cloudflare_access_application(app_id_to_delete_if_any):
             pass
     
-    reconcile_state_threaded() 
-    action_status_message += " Reconciliation triggered."
+    if source == "docker" and docker_client:
+        reconcile_state_threaded()
+        action_status_message += " Reconciliation triggered."
+    elif source == "agent":
+        agent = get_agent(agent_id)
+        containers = agent.get("last_complete_containers") if agent else None
+        if agent_inventory_contains_rule(containers, current_rule):
+            from app.core.reconciler import reconcile_agent_report
+            reconcile_agent_report(agent_id, containers)
+            action_status_message += " Agent reconciliation triggered."
+        else:
+            action_status_message += " Waiting for the next Agent report."
     cloudflared_agent_state["last_action_status"] = action_status_message
     return redirect(url_for('web.status_page'))
 
@@ -1113,12 +1245,14 @@ def reconciliation_status_route():
 
 @bp.route('/start-tunnel', methods=['POST'])
 def start_tunnel_route(): 
+    notification_manager.suppress_tunnel_health(tunnel_state.get("id"), 120)
     start_cloudflared_container() 
     time.sleep(1)
     return redirect(url_for('web.status_page'))
 
 @bp.route('/stop-tunnel', methods=['POST'])
 def stop_tunnel_route(): 
+    notification_manager.suppress_tunnel_health(tunnel_state.get("id"), 300)
     stop_cloudflared_container() 
     time.sleep(1)
     return redirect(url_for('web.status_page'))
@@ -1129,6 +1263,7 @@ def force_delete_rule_route(hostname):
     zone_id_for_delete = None
     access_app_id_for_delete = None
     tunnel_id_for_delete = None
+    source_for_delete = None
     fqdn = hostname.split('|')[0]
     with state_lock:
         rule_details = managed_rules.get(hostname)
@@ -1136,6 +1271,7 @@ def force_delete_rule_route(hostname):
             fqdn = rule_details.get("hostname") or fqdn
             zone_id_for_delete = rule_details.get("zone_id")
             access_app_id_for_delete = rule_details.get("access_app_id")
+            source_for_delete = rule_details.get("source") or "manual"
             tunnel_id_for_delete = rule_details.get("tunnel_id") or (tunnel_state.get("id") if not config.USE_EXTERNAL_CLOUDFLARED else config.EXTERNAL_TUNNEL_ID)
             del managed_rules[hostname]
             rule_removed_from_state = True
@@ -1148,12 +1284,29 @@ def force_delete_rule_route(hostname):
             and (rule.get("tunnel_id") or tunnel_id_for_delete) == tunnel_id_for_delete
             for rule in managed_rules.values()
         )
+    dns_deleted = True
+    access_deleted = True
+    tunnel_updated = True
     if zone_id_for_delete and tunnel_id_for_delete and not tuple_still_owned:
-        delete_cloudflare_dns_record(zone_id_for_delete, fqdn, tunnel_id_for_delete)
+        dns_deleted = bool(delete_cloudflare_dns_record(zone_id_for_delete, fqdn, tunnel_id_for_delete))
     if access_app_id_for_delete:
-        delete_cloudflare_access_application(access_app_id_for_delete)
+        access_deleted = bool(delete_cloudflare_access_application(access_app_id_for_delete))
     if rule_removed_from_state and not config.USE_EXTERNAL_CLOUDFLARED:
-        update_cloudflare_config(tunnel_id_for_delete)
+        tunnel_updated = bool(update_cloudflare_config(tunnel_id_for_delete))
+    if rule_removed_from_state:
+        context = {"hostname": fqdn, "operation": "force delete", "source": source_for_delete}
+        if not dns_deleted:
+            notification_manager.emit("cloudflare.dns_failure", f"{zone_id_for_delete}:{fqdn}", context)
+        if not access_deleted:
+            notification_manager.emit("cloudflare.access_failure", hostname, context)
+        if not tunnel_updated:
+            notification_manager.emit("cloudflare.tunnel_failure", str(tunnel_id_for_delete), context)
+        if dns_deleted and access_deleted and tunnel_updated:
+            notification_manager.emit(
+                "rule.deleted",
+                hostname,
+                {"source": source_for_delete, "resources": [{"key": hostname, "hostname": fqdn, "source": source_for_delete}], "public_url": config.DOCKFLARE_PUBLIC_URL},
+            )
     return redirect(url_for('web.status_page'))
 
 @bp.route('/stream-logs')
@@ -1565,10 +1718,6 @@ def ui_revert_docker_rule_route():
     """
     Reverts a UI-overridden Docker rule back to label-driven configuration.
     """
-    if not docker_client:
-        cloudflared_agent_state["last_action_status"] = "Error: Docker client unavailable."
-        return redirect(url_for('web.status_page'))
-
     rule_key = request.form.get('rule_key')
     if not rule_key:
         cloudflared_agent_state["last_action_status"] = "Error: Missing rule key for revert."
@@ -1580,26 +1729,39 @@ def ui_revert_docker_rule_route():
             cloudflared_agent_state["last_action_status"] = f"Error: Rule '{rule_key}' not found."
             return redirect(url_for('web.status_page'))
 
-        if existing.get("source") != "docker":
-            cloudflared_agent_state["last_action_status"] = f"Error: Rule '{rule_key}' is not a Docker rule."
+        source = existing.get("source")
+        if source not in {"docker", "agent"}:
+            cloudflared_agent_state["last_action_status"] = f"Error: Rule '{rule_key}' is not container-backed."
             return redirect(url_for('web.status_page'))
 
         if not existing.get("rule_ui_override", False):
             cloudflared_agent_state["last_action_status"] = f"Info: Rule '{rule_key}' is not UI-overridden."
             return redirect(url_for('web.status_page'))
 
-        # Revert the rule back to Docker label control
+        if source == "docker" and not docker_client:
+            cloudflared_agent_state["last_action_status"] = "Error: Docker client unavailable."
+            return redirect(url_for('web.status_page'))
+
         existing["rule_ui_override"] = False
+        existing["lifecycle_generation"] = int(existing.get("lifecycle_generation") or 0) + 1
         save_state()
 
-        cloudflared_agent_state["last_action_status"] = f"Success: Rule '{rule_key}' reverted to Docker label control. Reconciliation will update it based on container labels."
+        cloudflared_agent_state["last_action_status"] = f"Success: Rule '{rule_key}' reverted to label control."
 
-    # Trigger reconciliation to pick up Docker labels
     try:
-        from app.core.reconciler import reconcile_state_threaded
-        reconcile_state_threaded()
+        if source == "docker":
+            from app.core.reconciler import reconcile_state_threaded
+            reconcile_state_threaded()
+        else:
+            from app.core.reconciler import reconcile_agent_report
+            agent = get_agent(existing.get("agent_id"))
+            containers = agent.get("last_complete_containers") if agent else None
+            if agent_inventory_contains_rule(containers, existing):
+                reconcile_agent_report(existing.get("agent_id"), containers)
+            else:
+                cloudflared_agent_state["last_action_status"] += " Waiting for the next Agent report."
     except Exception as e:
-        logging.error(f"Failed to trigger reconciliation after Docker rule revert: {e}")
+        logging.error(f"Failed to trigger reconciliation after rule revert: {e}")
 
     return redirect(url_for('web.status_page'))
 
@@ -1610,10 +1772,6 @@ def ui_edit_manual_rule_route():
     Expects a hidden 'edit_rule_key' form field identifying the rule key to edit.
     Only rules with source != 'docker' (manual or agent) are editable via UI.
     """
-    if not docker_client:
-        cloudflared_agent_state["last_action_status"] = "Error: Docker client unavailable."
-        return redirect(url_for('web.status_page'))
-
     rule_key = request.form.get('edit_rule_key')
     if not rule_key:
         cloudflared_agent_state["last_action_status"] = "Error: Missing rule key for edit."
@@ -1625,7 +1783,7 @@ def ui_edit_manual_rule_route():
             cloudflared_agent_state["last_action_status"] = f"Error: Rule '{rule_key}' not found."
             return redirect(url_for('web.status_page'))
         existing = copy.deepcopy(existing)
-        is_docker_rule = existing.get("source") == "docker"
+        is_container_backed = existing.get("source") in {"docker", "agent"}
     
     subdomain_input = request.form.get('edit_subdomain', '').strip()
     domain_name_input = request.form.get('edit_domain_name', '').strip()
@@ -1831,12 +1989,13 @@ def ui_edit_manual_rule_route():
             "access_app_config_hash": access_app_config_hash,
             "access_group_id": access_group_id,
             "access_policy_ui_override": True,
-            "rule_ui_override": is_docker_rule,
+            "rule_ui_override": is_container_backed,
             "source": existing.get("source", "manual"),
             "agent_id": existing.get("agent_id"),
             "tunnel_id": target_tunnel_id,
             "tunnel_name": target_tunnel_name
         })
+    rule_entry["lifecycle_generation"] = int(existing.get("lifecycle_generation") or 0) + 1
 
     with state_lock:
         conflicting = managed_rules.get(new_key)
@@ -2003,7 +2162,10 @@ def create_access_group():
             "policies": policies
         }
         access_groups[group_id] = new_group
-        save_state()
+        if not save_state():
+            access_groups.pop(group_id, None)
+            flash(_t('flash.access_group.create_error', error='persistence_failed'), "error")
+            return redirect(url_for('web.access_policies_page'))
 
         from app import config
         if config.USE_REUSABLE_POLICIES and len(effective_access_policies(new_group)) == 1:
@@ -2017,6 +2179,19 @@ def create_access_group():
 
         publish_state_event('snapshot_refresh')
 
+    notification_manager.emit(
+        "access.policy_created",
+        group_id,
+        {
+            "policy_name": display_name,
+            "policy_id": group_id,
+            "policy_type": "Reusable" if config.USE_REUSABLE_POLICIES else "Inline",
+            "identity_provider_count": len(new_group.get("allowed_idps") or []),
+            "rules_count": len(effective_access_policies(new_group)),
+            "source": "admin",
+            "public_url": config.DOCKFLARE_PUBLIC_URL,
+        },
+    )
     flash(_t('flash.access_group.created', displayName=display_name), "success")
     return redirect(url_for('web.access_policies_page'))
 
@@ -2057,6 +2232,7 @@ def edit_access_group(group_id):
             flash(_t('flash.access_group.update_error', error=str(e)), "error")
             return redirect(url_for('web.access_policies_page'))
 
+        previous_group = copy.deepcopy(access_groups[group_id])
         updated_group = {
             "id": group_id,
             "display_name": display_name,
@@ -2068,7 +2244,20 @@ def edit_access_group(group_id):
             "policies": policies
         }
         access_groups[group_id] = updated_group
-        save_state()
+        if not save_state():
+            access_groups[group_id] = previous_group
+            flash(_t('flash.access_group.update_error', error='persistence_failed'), "error")
+            return redirect(url_for('web.access_policies_page'))
+
+        affected_service_count = sum(
+            1
+            for rule in managed_rules.values()
+            if group_id in (
+                rule.get('access_group_id')
+                if isinstance(rule.get('access_group_id'), list)
+                else [rule.get('access_group_id')]
+            )
+        )
 
         from app import config
         if config.USE_REUSABLE_POLICIES and len(effective_access_policies(updated_group)) == 1:
@@ -2082,6 +2271,20 @@ def edit_access_group(group_id):
 
         publish_state_event('snapshot_refresh')
 
+    notification_manager.emit(
+        "access.policy_updated",
+        group_id,
+        {
+            "policy_name": display_name,
+            "policy_id": group_id,
+            "policy_type": "Reusable" if config.USE_REUSABLE_POLICIES else "Inline",
+            "identity_provider_count": len(updated_group.get("allowed_idps") or []),
+            "rules_count": len(effective_access_policies(updated_group)),
+            "affected_service_count": affected_service_count,
+            "source": "admin",
+            "public_url": config.DOCKFLARE_PUBLIC_URL,
+        },
+    )
     flash(_t('flash.access_group.updated', displayName=display_name), "success")
     reconcile_state_threaded()
     return redirect(url_for('web.access_policies_page'))
@@ -2107,24 +2310,46 @@ def delete_access_group(group_id):
             flash(_t('flash.access_group.delete_in_use', displayName=access_groups[group_id]['display_name']), "error")
             return redirect(url_for('web.access_policies_page'))
 
-        display_name = access_groups[group_id]['display_name']
+        previous_group = copy.deepcopy(access_groups[group_id])
+        display_name = previous_group['display_name']
 
         from app import config
+        deletion_persisted = True
         if config.USE_REUSABLE_POLICIES:
             from app.core import reusable_policies
             try:
-                reusable_policies.delete_access_group_and_policy(group_id)
-                logging.info(f"Deleted access group '{group_id}' and associated Cloudflare policy")
+                deletion_persisted = reusable_policies.delete_access_group_and_policy(group_id)
+                if deletion_persisted:
+                    logging.info(f"Deleted access group '{group_id}' and associated Cloudflare policy")
             except Exception as e:
                 logging.error(f"Error deleting Cloudflare policy for '{group_id}': {e}", exc_info=True)
                 del access_groups[group_id]
-                save_state()
+                deletion_persisted = save_state()
+                if not deletion_persisted:
+                    access_groups[group_id] = previous_group
         else:
             del access_groups[group_id]
-            save_state()
+            deletion_persisted = save_state()
+            if not deletion_persisted:
+                access_groups[group_id] = previous_group
+
+        if not deletion_persisted:
+            flash(_t('flash.access_group.update_error', error='persistence_failed'), "error")
+            return redirect(url_for('web.access_policies_page'))
 
         publish_state_event('snapshot_refresh')
 
+    notification_manager.emit(
+        "access.policy_deleted",
+        group_id,
+        {
+            "policy_name": display_name,
+            "policy_id": group_id,
+            "policy_type": "Reusable" if config.USE_REUSABLE_POLICIES else "Inline",
+            "source": "admin",
+            "public_url": config.DOCKFLARE_PUBLIC_URL,
+        },
+    )
     flash(_t('flash.access_group.deleted', displayName=display_name), "success")
     return redirect(url_for('web.access_policies_page'))
 

dockflare/package-lock.json

@@ -13,7 +13,7 @@
       "devDependencies": {
         "autoprefixer": "^10.4.19",
         "daisyui": "^4.10.2",
-        "postcss": "^8.4.38",
+        "postcss": "^8.5.26",
         "tailwindcss": "^3.4.17"
       }
     },
@@ -49,9 +49,9 @@
       }
     },
     "node_modules/@jridgewell/sourcemap-codec": {
-      "version": "1.5.5",
-      "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz",
-      "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==",
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz",
+      "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==",
       "license": "MIT"
     },
     "node_modules/@jridgewell/trace-mapping": {
@@ -174,9 +174,9 @@
       }
     },
     "node_modules/baseline-browser-mapping": {
-      "version": "2.11.7",
-      "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.7.tgz",
-      "integrity": "sha512-APw5YuIQAg6L9w4sHDI6j26DGFJI6RpYOhnkMPdC9lWbkKvsyPHzDsve1yd73lk21yz7Y09Kci8B2Pp9FonzWA==",
+      "version": "2.11.20",
+      "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.20.tgz",
+      "integrity": "sha512-H0ulySigv6icDJ1F7SjtdCD6PrhTpdYCmP0CactWy1+ekh0AFd0o1Wn5T8b+hnTmdBx19u9yhL6wvCylXMY7zw==",
       "dev": true,
       "license": "Apache-2.0",
       "bin": {
@@ -211,9 +211,9 @@
       }
     },
     "node_modules/browserslist": {
-      "version": "4.28.7",
-      "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.7.tgz",
-      "integrity": "sha512-JxV13hNrFxqjOc8alRbq9dK1MM79NEXYpma2B2J4wAtpWS5zIEIKqWPGCl7N4o7Uc7B7itylh7SuDujATRyyTw==",
+      "version": "4.28.8",
+      "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.8.tgz",
+      "integrity": "sha512-V2NpofLblG64mfOtSgDhOJESZEGogzDMBv/q+W6oc4LXWP/q75eOXoOaaOu1EOadB9U4Bwx/e0yzbvwKH8zalA==",
       "dev": true,
       "funding": [
         {
@@ -231,11 +231,11 @@
       ],
       "license": "MIT",
       "dependencies": {
-        "baseline-browser-mapping": "^2.10.44",
-        "caniuse-lite": "^1.0.30001806",
-        "electron-to-chromium": "^1.5.393",
-        "node-releases": "^2.0.51",
-        "update-browserslist-db": "^1.2.3"
+        "baseline-browser-mapping": "^2.11.12",
+        "caniuse-lite": "^1.0.30001809",
+        "electron-to-chromium": "^1.5.402",
+        "node-releases": "^2.0.53",
+        "update-browserslist-db": "^1.3.0"
       },
       "bin": {
         "browserslist": "cli.js"
@@ -254,9 +254,9 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001806",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001806.tgz",
-      "integrity": "sha512-72Cuvd95zbSYPKq6Fhg8eDJRlzgWDf7/mtoZv6Qe/DYNCEBdNxoA3+rZAU2ZhGCpZlns3EssFavaZomckT5Uuw==",
+      "version": "1.0.30001810",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz",
+      "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==",
       "dev": true,
       "funding": [
         {
@@ -385,9 +385,9 @@
       "license": "MIT"
     },
     "node_modules/electron-to-chromium": {
-      "version": "1.5.398",
-      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.398.tgz",
-      "integrity": "sha512-AsvhAxopJGh6museTDMIjn6JpDYOfgu4RLlygomt87MUwBUqTfd/1EiPtx10/LZE8xpTvkP2E9Gafq7lkLtodQ==",
+      "version": "1.5.416",
+      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.416.tgz",
+      "integrity": "sha512-K6bvB2BjnNrugtIih6ewlbBI9DXa976jIdiIlRLHhBoEI9a4JaQjjHyF+A1IQI543aQYR4LnmOrT/K5fZj0aPA==",
       "dev": true,
       "license": "ISC"
     },
@@ -446,9 +446,9 @@
       "license": "MIT"
     },
     "node_modules/fastq": {
-      "version": "1.20.1",
-      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz",
-      "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==",
+      "version": "1.20.2",
+      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.2.tgz",
+      "integrity": "sha512-UpGiiODyCGprM8EPP6JodP6jC9Rws6TCuiDOD+nn0CJhR8guI3g/ozo4ugL0vJ+Yz1UtJuuRPqvQuybVOF1VQA==",
       "license": "ISC",
       "dependencies": {
         "reusify": "^1.0.4"
@@ -680,9 +680,9 @@
       }
     },
     "node_modules/node-releases": {
-      "version": "2.0.51",
-      "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.51.tgz",
-      "integrity": "sha512-wRNIrw4DmVLKQlbgOMdkMx27Wrpzes2hh5Jtbi2bjPd+4wJstWIqP5A+lscnqbm0xxmT5Bpg8Lec5ItEBwx6BQ==",
+      "version": "2.0.54",
+      "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.54.tgz",
+      "integrity": "sha512-YHs7BmmcsdAI5Ozuf8JZo6PT0mv2GIWC9vMfvUC3dp65M8hn7Ux8CPL+2oBI7juNuj9d0ndhTcznq2ODBps9cQ==",
       "dev": true,
       "license": "MIT",
       "engines": {
@@ -740,15 +740,6 @@
         "url": "https://github.com/sponsors/jonschlinkert"
       }
     },
-    "node_modules/pify": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
-      "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/pirates": {
       "version": "4.0.7",
       "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz",
@@ -759,9 +750,9 @@
       }
     },
     "node_modules/postcss": {
-      "version": "8.5.25",
-      "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz",
-      "integrity": "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==",
+      "version": "8.5.26",
+      "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz",
+      "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==",
       "funding": [
         {
           "type": "opencollective",
@@ -778,7 +769,7 @@
       ],
       "license": "MIT",
       "dependencies": {
-        "nanoid": "^3.3.16",
+        "nanoid": "^3.3.17",
         "picocolors": "^1.1.1",
         "source-map-js": "^1.2.1"
       },
@@ -948,13 +939,10 @@
       "license": "MIT"
     },
     "node_modules/read-cache": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz",
-      "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==",
-      "license": "MIT",
-      "dependencies": {
-        "pify": "^2.3.0"
-      }
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.2.tgz",
+      "integrity": "sha512-/peqiBB/n07gQGLsWaHho3WfvUyRscw0gYTsEFMhrIe/nWLkYaf5SbKYjGYqtRV3aPwykJgF2VEMo1ac4bnsGA==",
+      "license": "MIT"
     },
     "node_modules/readdirp": {
       "version": "3.6.0",
@@ -1171,9 +1159,9 @@
       "license": "Apache-2.0"
     },
     "node_modules/update-browserslist-db": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz",
-      "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==",
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.2.tgz",
+      "integrity": "sha512-UQ+MSxlhRm1bzjhU+DcuXfjFO1FzNtqhK5+9Yvlp90ItDLk5vT932A0rFu619nf7RVS+Y/VeaUW1jaRDqZ8VJw==",
       "dev": true,
       "funding": [
         {

dockflare/package.json

@@ -13,7 +13,7 @@
   "devDependencies": {
     "autoprefixer": "^10.4.19",
     "daisyui": "^4.10.2",
-    "postcss": "^8.4.38",
+    "postcss": "^8.5.26",
     "tailwindcss": "^3.4.17"
   },
   "dependencies": {

dockflare/requirements.in

@@ -1,5 +1,5 @@
 Authlib==1.6.11
-boto3
+boto3==1.42.83
 cryptography==46.0.7
 docker==7.1.0
 Flask==3.1.3
@@ -8,6 +8,7 @@ Flask-Limiter==3.13
 Flask-Login==0.6.3
 Flask-WTF==1.2.2
 Markdown==3.8.1
+apprise==1.13.1
 Pygments==2.20.0
 PyJWT[crypto]>=2.8.0
 pymdown-extensions==10.21.2

dockflare/requirements.txt

@@ -4,34 +4,37 @@
 #
 #    pip-compile --output-file=dockflare/requirements.txt dockflare/requirements.in
 #
+apprise==1.13.1
+    # via -r dockflare/requirements.in
 authlib==1.6.11
     # via -r dockflare/requirements.in
+blinker==1.9.0
+    # via flask
 boto3==1.42.83
     # via -r dockflare/requirements.in
 botocore==1.42.88
-    # via boto3
-jmespath==1.1.0
-    # via boto3, botocore
-python-dateutil==2.9.0.post0
-    # via botocore
-s3transfer==0.16.0
-    # via boto3
-blinker==1.9.0
-    # via flask
+    # via
+    #   boto3
+    #   s3transfer
 cachelib==0.9.0
     # via flask-caching
 certifi==2025.8.3
-    # via requests
+    # via
+    #   apprise
+    #   requests
 cffi==2.0.0
     # via cryptography
 charset-normalizer==3.4.2
     # via requests
 click==8.2.1
-    # via flask
+    # via
+    #   apprise
+    #   flask
 cryptography==46.0.7
     # via
     #   -r dockflare/requirements.in
     #   authlib
+    #   pyjwt
 deprecated==1.3.1
     # via limits
 docker==7.1.0
@@ -59,11 +62,16 @@ itsdangerous==2.2.0
     #   flask-wtf
 jinja2==3.1.6
     # via flask
+jmespath==1.1.0
+    # via
+    #   boto3
+    #   botocore
 limits==5.8.0
     # via flask-limiter
 markdown==3.8.1
     # via
     #   -r dockflare/requirements.in
+    #   apprise
     #   pymdown-extensions
 markdown-it-py==4.0.0
     # via rich
@@ -75,6 +83,8 @@ markupsafe==3.0.2
     #   wtforms
 mdurl==0.1.2
     # via markdown-it-py
+oauthlib==3.3.1
+    # via requests-oauthlib
 ordered-set==4.1.0
     # via flask-limiter
 packaging==26.0
@@ -85,20 +95,36 @@ pygments==2.20.0
     # via
     #   -r dockflare/requirements.in
     #   rich
+pyjwt[crypto]==2.12.1
+    # via
+    #   -r dockflare/requirements.in
+    #   pyjwt
 pymdown-extensions==10.21.2
     # via -r dockflare/requirements.in
+python-dateutil==2.9.0.post0
+    # via botocore
 python-dotenv==1.2.2
     # via -r dockflare/requirements.in
 pyyaml==6.0.3
-    # via pymdown-extensions
+    # via
+    #   apprise
+    #   pymdown-extensions
 redis==4.5.5
     # via -r dockflare/requirements.in
 requests==2.33.1
     # via
     #   -r dockflare/requirements.in
+    #   apprise
     #   docker
+    #   requests-oauthlib
+requests-oauthlib==2.0.0
+    # via apprise
 rich==14.3.3
     # via flask-limiter
+s3transfer==0.16.0
+    # via boto3
+six==1.17.0
+    # via python-dateutil
 typing-extensions==4.15.0
     # via
     #   flask-limiter
@@ -106,6 +132,7 @@ typing-extensions==4.15.0
 urllib3==2.6.3
     # via
     #   -r dockflare/requirements.in
+    #   botocore
     #   docker
     #   requests
 waitress==3.0.2
@@ -119,5 +146,3 @@ wrapt==2.1.2
     # via deprecated
 wtforms==3.2.1
     # via flask-wtf
-pyjwt[crypto]==2.12.1
-    # via -r dockflare/requirements.in

dockflare/tests/test_access_policy_rules.py

@@ -1,154 +0,0 @@
-import importlib.util
-import pathlib
-import unittest
-
-
-MODULE_PATH = pathlib.Path(__file__).parents[1] / "app" / "core" / "access_policy_rules.py"
-SPEC = importlib.util.spec_from_file_location("access_policy_rules", MODULE_PATH)
-RULES = importlib.util.module_from_spec(SPEC)
-SPEC.loader.exec_module(RULES)
-
-
-class AccessPolicyRuleTests(unittest.TestCase):
-    def setUp(self):
-        self.idps = {
-            "google": "google-id",
-            "github": "github-id"
-        }
-
-    def resolve_idp(self, name):
-        return self.idps.get(name)
-
-    def test_reserved_system_groups_are_read_only_without_metadata(self):
-        self.assertTrue(RULES.is_system_access_group("authenticated-default", {}))
-        self.assertTrue(RULES.is_system_access_group("public-default-bypass", {}))
-
-    def test_explicit_system_group_is_read_only(self):
-        self.assertTrue(RULES.is_system_access_group("legacy-system", {"system_policy": True}))
-
-    def test_custom_group_is_not_system_owned(self):
-        self.assertFalse(RULES.is_system_access_group("google-users", {"system_policy": False}))
-
-    def test_multiple_emails_are_or_within_selected_idp(self):
-        policies = RULES.build_access_policies(
-            "alice@gmail.com, bob@gmail.com, @example.com",
-            idp_list=["google"],
-            idp_resolver=self.resolve_idp
-        )
-
-        self.assertEqual(policies[0]["include"], [
-            {"email": {"email": "alice@gmail.com"}},
-            {"email": {"email": "bob@gmail.com"}},
-            {"email_domain": {"domain": "example.com"}}
-        ])
-        self.assertEqual(policies[0]["require"], [
-            {"login_method": {"id": "google-id"}}
-        ])
-        self.assertEqual(policies[1]["decision"], "deny")
-
-    def test_multiple_idps_create_alternative_allow_policies(self):
-        policies = RULES.build_access_policies(
-            "alice@example.com, bob@example.com",
-            idp_list=["google", "github"],
-            idp_resolver=self.resolve_idp
-        )
-
-        allow_policies = [policy for policy in policies if policy["decision"] == "allow"]
-        self.assertEqual(len(allow_policies), 2)
-        self.assertEqual(allow_policies[0]["include"], allow_policies[1]["include"])
-        self.assertEqual(allow_policies[0]["require"], [{"login_method": {"id": "google-id"}}])
-        self.assertEqual(allow_policies[1]["require"], [{"login_method": {"id": "github-id"}}])
-
-    def test_missing_idp_fails_closed(self):
-        with self.assertRaisesRegex(ValueError, "missing"):
-            RULES.build_access_policies(
-                "alice@example.com",
-                idp_list=["google", "missing"],
-                idp_resolver=self.resolve_idp
-            )
-
-    def test_idp_without_email_is_rejected(self):
-        with self.assertRaisesRegex(ValueError, "allowed email"):
-            RULES.build_access_policies(
-                "",
-                idp_list=["google"],
-                idp_resolver=self.resolve_idp
-            )
-
-    def test_ip_bypass_and_authentication_remain_separate(self):
-        policies = RULES.build_access_policies(
-            "alice@example.com",
-            ip_ranges_str="192.0.2.0/24",
-            idp_list=["google"],
-            idp_resolver=self.resolve_idp
-        )
-
-        effective = RULES.effective_access_policies({"policies": policies})
-        self.assertEqual([policy["decision"] for policy in effective], ["bypass", "allow"])
-
-    def test_login_method_ids_reads_include_and_require(self):
-        policies = [
-            {
-                "include": [{"login_method": {"id": "legacy-id"}}],
-                "require": [{"login_method": {"id": "required-id"}}]
-            }
-        ]
-
-        self.assertEqual(RULES.login_method_ids(policies), ["legacy-id", "required-id"])
-
-    def test_legacy_or_policy_is_migrated(self):
-        group = {
-            "policies": [
-                {
-                    "name": "Allow defined users",
-                    "decision": "allow",
-                    "include": [
-                        {"email": {"email": "alice@example.com"}},
-                        {"email": {"email": "bob@example.com"}},
-                        {"login_method": {"id": "google-id"}}
-                    ]
-                },
-                {"name": "Default Deny", "decision": "deny", "include": [{"everyone": {}}]}
-            ]
-        }
-
-        normalized, changed = RULES.normalize_managed_access_group(group)
-
-        self.assertTrue(changed)
-        self.assertEqual(normalized["policies"][0]["include"], [
-            {"email": {"email": "alice@example.com"}},
-            {"email": {"email": "bob@example.com"}}
-        ])
-        self.assertEqual(normalized["policies"][0]["require"], [
-            {"login_method": {"id": "google-id"}}
-        ])
-
-    def test_inverted_require_policy_is_migrated(self):
-        group = {
-            "policies": [
-                {
-                    "name": "Allow defined users",
-                    "decision": "allow",
-                    "include": [{"login_method": {"id": "google-id"}}],
-                    "require": [
-                        {"email": {"email": "alice@example.com"}},
-                        {"email": {"email": "bob@example.com"}}
-                    ]
-                }
-            ]
-        }
-
-        normalized, changed = RULES.normalize_managed_access_group(group)
-
-        self.assertTrue(changed)
-        self.assertEqual(normalized["policies"][0]["include"], [
-            {"email": {"email": "alice@example.com"}},
-            {"email": {"email": "bob@example.com"}}
-        ])
-        self.assertEqual(normalized["policies"][0]["require"], [
-            {"login_method": {"id": "google-id"}}
-        ])
-
-
-if __name__ == "__main__":
-    unittest.main()

dockflare/tests/test_agent_tunnel_config.py

@@ -1,164 +0,0 @@
-import copy
-import unittest
-from unittest.mock import patch
-
-from app import app
-from app.core import tunnel_manager
-from app.core.state_manager import agents, managed_rules, state_lock
-from app.web import api_v2_routes
-
-
-class AgentTunnelConfigTests(unittest.TestCase):
-    def setUp(self):
-        with state_lock:
-            self.rules_snapshot = copy.deepcopy(managed_rules)
-            self.agents_snapshot = copy.deepcopy(agents)
-            managed_rules.clear()
-            agents.clear()
-
-    def tearDown(self):
-        with state_lock:
-            managed_rules.clear()
-            managed_rules.update(self.rules_snapshot)
-            agents.clear()
-            agents.update(self.agents_snapshot)
-
-    def test_agent_event_preserves_manual_rules_on_the_same_tunnel(self):
-        with state_lock:
-            agents["agent-1"] = {
-                "id": "agent-1",
-                "assigned_tunnel_id": "agent-tunnel",
-                "assigned_tunnel_name": "Agent Tunnel"
-            }
-            managed_rules["plex.example.com|"] = {
-                "hostname": "plex.example.com",
-                "service": "https://192.0.2.10:32400",
-                "status": "active",
-                "source": "manual",
-                "tunnel_id": "agent-tunnel",
-                "no_tls_verify": True
-            }
-            managed_rules["other.example.com|"] = {
-                "hostname": "other.example.com",
-                "service": "http://192.0.2.20:8080",
-                "status": "active",
-                "source": "manual",
-                "tunnel_id": "other-tunnel"
-            }
-
-        payload = {
-            "container": {
-                "id": "container-1",
-                "name": "sonarr",
-                "labels": {
-                    "dockflare.enable": "true",
-                    "dockflare.hostname": "sonarr.example.com",
-                    "dockflare.service": "http://sonarr:8989"
-                }
-            }
-        }
-        requests = []
-
-        def capture_request(method, endpoint, json_data=None, **kwargs):
-            requests.append((method, endpoint, json_data))
-            return {"success": True}
-
-        with app.app_context():
-            with patch.dict(app.config, {"CF_ZONE_ID": "zone-1"}):
-                with patch.object(api_v2_routes, "save_state"), \
-                     patch.object(api_v2_routes, "publish_state_event"), \
-                     patch.object(api_v2_routes, "get_account_zone_inventory", return_value={"zones": [{"id": "zone-1", "name": "example.com"}], "status": "complete", "error": None}), \
-                     patch.object(api_v2_routes, "create_cloudflare_dns_record"), \
-                     patch.object(api_v2_routes, "handle_access_policy_from_labels", return_value=False), \
-                     patch.object(tunnel_manager.cloudflare_api, "get_current_cf_config", return_value={"ingress": []}), \
-                     patch.object(tunnel_manager.cloudflare_api, "cf_api_request", side_effect=capture_request):
-                    api_v2_routes.process_agent_container_start(payload, "agent-1")
-
-        self.assertEqual(len(requests), 1)
-        method, endpoint, body = requests[0]
-        self.assertEqual(method, "PUT")
-        self.assertIn("/agent-tunnel/configurations", endpoint)
-
-        ingress = body["config"]["ingress"]
-        entries = {entry.get("hostname"): entry for entry in ingress if entry.get("hostname")}
-        self.assertEqual(set(entries), {"plex.example.com", "sonarr.example.com"})
-        self.assertEqual(entries["plex.example.com"]["originRequest"], {"noTLSVerify": True})
-        self.assertEqual(ingress[-1], {"service": "http_status:404"})
-        with state_lock:
-            agent_rule = managed_rules["sonarr.example.com|"]
-            self.assertEqual(agent_rule["zone_id"], "zone-1")
-            self.assertEqual(agent_rule["zone_name"], "example.com")
-            self.assertEqual(agent_rule["zone_resolution_source"], "hostname")
-            self.assertEqual(agent_rule["agent_id"], "agent-1")
-            self.assertEqual(agent_rule["tunnel_id"], "agent-tunnel")
-
-    def test_agent_tunnel_builder_excludes_other_tunnels_and_inactive_rules(self):
-        with state_lock:
-            agents["agent-1"] = {"assigned_tunnel_id": "agent-tunnel"}
-            agents["agent-2"] = {"assigned_tunnel_id": "other-tunnel"}
-            managed_rules["agent.example.com|"] = {
-                "hostname": "agent.example.com",
-                "service": "http://agent:8080",
-                "status": "active",
-                "source": "agent",
-                "agent_id": "agent-1",
-                "tunnel_id": "agent-tunnel"
-            }
-            managed_rules["inactive.example.com|"] = {
-                "hostname": "inactive.example.com",
-                "service": "http://inactive:8080",
-                "status": "pending_deletion",
-                "source": "agent",
-                "agent_id": "agent-1",
-                "tunnel_id": "agent-tunnel"
-            }
-            managed_rules["other.example.com|"] = {
-                "hostname": "other.example.com",
-                "service": "http://other:8080",
-                "status": "active",
-                "source": "agent",
-                "agent_id": "agent-2",
-                "tunnel_id": "other-tunnel"
-            }
-
-        ingress = tunnel_manager._build_ingress_entries_for_tunnel("agent-tunnel", "master-tunnel")
-        hostnames = [entry.get("hostname") for entry in ingress if entry.get("hostname")]
-
-        self.assertEqual(hostnames, ["agent.example.com"])
-        self.assertEqual(ingress[-1], {"service": "http_status:404"})
-
-    def test_agent_invalid_explicit_zone_fails_closed(self):
-        with state_lock:
-            agents["agent-1"] = {
-                "id": "agent-1",
-                "assigned_tunnel_id": "agent-tunnel",
-                "assigned_tunnel_name": "Agent Tunnel"
-            }
-        payload = {
-            "container": {
-                "id": "container-2",
-                "name": "invalid-zone",
-                "labels": {
-                    "dockflare.enable": "true",
-                    "dockflare.hostname": "app.example.com",
-                    "dockflare.service": "http://app:8080",
-                    "dockflare.zonename": "invalid.example"
-                }
-            }
-        }
-        inventory = {"zones": [{"id": "zone-1", "name": "example.com"}], "status": "complete", "error": None}
-        with app.app_context():
-            with patch.object(api_v2_routes, "get_account_zone_inventory", return_value=inventory), \
-                 patch.object(api_v2_routes, "save_state"), \
-                 patch.object(api_v2_routes, "publish_state_event"), \
-                 patch.object(api_v2_routes, "create_cloudflare_dns_record") as create_dns, \
-                 patch.object(api_v2_routes, "update_cloudflare_config") as update_tunnel:
-                api_v2_routes.process_agent_container_start(payload, "agent-1")
-        with state_lock:
-            self.assertNotIn("app.example.com|", managed_rules)
-        create_dns.assert_not_called()
-        update_tunnel.assert_not_called()
-
-
-if __name__ == "__main__":
-    unittest.main()

dockflare/tests/test_cache_pattern_invalidation.py

@@ -1,41 +0,0 @@
-import logging
-
-from app.core import cache as cache_module
-from flask import Flask
-from flask_caching import Cache
-
-
-def make_simple_cache():
-    app = Flask(__name__)
-    simple_cache = Cache(config={"CACHE_TYPE": "SimpleCache", "CACHE_DEFAULT_TIMEOUT": 300})
-    simple_cache.init_app(app)
-    return simple_cache
-
-
-def test_delete_keys_by_pattern_removes_matching_simple_cache_keys(monkeypatch):
-    simple_cache = make_simple_cache()
-    simple_cache.set("dns_records:zone-a:tunnel-1", "a")
-    simple_cache.set("dns_records:zone-a:tunnel-2", "b")
-    simple_cache.set("dns_records:zone-b:tunnel-1", "c")
-    simple_cache.set("zone_details:zone-a", "keep")
-
-    monkeypatch.setattr(cache_module, "cache", simple_cache)
-
-    cache_module._delete_keys_by_pattern("dns_records:zone-a:*")
-
-    assert simple_cache.get("dns_records:zone-a:tunnel-1") is None
-    assert simple_cache.get("dns_records:zone-a:tunnel-2") is None
-    assert simple_cache.get("dns_records:zone-b:tunnel-1") == "c"
-    assert simple_cache.get("zone_details:zone-a") == "keep"
-
-
-def test_simple_cache_pattern_invalidation_does_not_emit_redis_warning(monkeypatch, caplog):
-    simple_cache = make_simple_cache()
-    simple_cache.set("dns_records:zone-a:tunnel-1", "a")
-    monkeypatch.setattr(cache_module, "cache", simple_cache)
-
-    with caplog.at_level(logging.WARNING):
-        cache_module._delete_keys_by_pattern("dns_records:*")
-
-    assert "Pattern-based cache invalidation is only supported with RedisCache" not in caplog.text
-    assert simple_cache.get("dns_records:zone-a:tunnel-1") is None

dockflare/tests/test_zone_inventory.py

@@ -1,65 +0,0 @@
-import unittest
-from unittest.mock import patch
-
-import requests
-
-from app import app
-from app.core import cloudflare_api
-
-
-class ZoneInventoryTests(unittest.TestCase):
-    def setUp(self):
-        cloudflare_api._last_good_zone_inventories.clear()
-
-    def tearDown(self):
-        cloudflare_api._last_good_zone_inventories.clear()
-
-    def test_inventory_uses_supported_pagination(self):
-        first_page = [{"id": f"zone-{index}", "name": f"zone-{index}.example"} for index in range(50)]
-        second_page = [{"id": "zone-50", "name": "zone-50.example"}]
-        responses = [
-            {"result": first_page, "result_info": {"total_pages": 2}},
-            {"result": second_page, "result_info": {"total_pages": 2}},
-        ]
-        with app.app_context(), patch.dict(app.config, {"CF_ACCOUNT_ID": "account-1"}), \
-             patch.object(cloudflare_api.cache, "get", return_value=None), \
-             patch.object(cloudflare_api.cache, "set") as cache_set, \
-             patch.object(cloudflare_api, "cf_api_request", side_effect=responses) as request:
-            inventory = cloudflare_api.get_account_zone_inventory(force_refresh=True)
-        self.assertEqual(inventory["status"], "complete")
-        self.assertEqual(len(inventory["zones"]), 51)
-        self.assertEqual(request.call_count, 2)
-        self.assertEqual(request.call_args_list[0].kwargs["params"]["per_page"], 50)
-        self.assertEqual(request.call_args_list[1].kwargs["params"]["page"], 2)
-        cache_set.assert_called_once()
-
-    def test_partial_refresh_keeps_last_known_good_inventory(self):
-        previous = [{"id": "zone-1", "name": "example.com"}]
-        cloudflare_api._last_good_zone_inventories["account-1"] = previous
-        first_page = [{"id": f"zone-{index}", "name": f"zone-{index}.example"} for index in range(50)]
-        responses = [
-            {"result": first_page, "result_info": {"total_pages": 2}},
-            requests.exceptions.RequestException("upstream failure"),
-        ]
-        with app.app_context(), patch.dict(app.config, {"CF_ACCOUNT_ID": "account-1"}), \
-             patch.object(cloudflare_api.cache, "get", return_value=None), \
-             patch.object(cloudflare_api.cache, "set") as cache_set, \
-             patch.object(cloudflare_api, "cf_api_request", side_effect=responses):
-            inventory = cloudflare_api.get_account_zone_inventory(force_refresh=True)
-        self.assertEqual(inventory["status"], "stale")
-        self.assertEqual(inventory["zones"], previous)
-        cache_set.assert_not_called()
-
-    def test_failed_first_load_is_not_cached_as_empty_success(self):
-        with app.app_context(), patch.dict(app.config, {"CF_ACCOUNT_ID": "account-1"}), \
-             patch.object(cloudflare_api.cache, "get", return_value=None), \
-             patch.object(cloudflare_api.cache, "set") as cache_set, \
-             patch.object(cloudflare_api, "cf_api_request", side_effect=requests.exceptions.RequestException("upstream failure")):
-            inventory = cloudflare_api.get_account_zone_inventory(force_refresh=True)
-        self.assertEqual(inventory["status"], "unavailable")
-        self.assertEqual(inventory["zones"], [])
-        cache_set.assert_not_called()
-
-
-if __name__ == "__main__":
-    unittest.main()

dockflare/tests/test_zone_resolver.py

@@ -1,101 +0,0 @@
-import importlib.util
-import pathlib
-import unittest
-
-
-MODULE_PATH = pathlib.Path(__file__).parents[1] / "app" / "core" / "zone_resolver.py"
-SPEC = importlib.util.spec_from_file_location("zone_resolver", MODULE_PATH)
-RESOLVER = importlib.util.module_from_spec(SPEC)
-SPEC.loader.exec_module(RESOLVER)
-
-
-class ZoneResolverTests(unittest.TestCase):
-    def setUp(self):
-        self.zones = [
-            {"id": "parent", "name": "side.co.uk"},
-            {"id": "nested", "name": "internal.side.co.uk"},
-            {"id": "primary", "name": "example.net"},
-        ]
-
-    def test_multi_label_zone(self):
-        result = RESOLVER.match_zone_for_hostname("domain.side.co.uk", self.zones)
-        self.assertEqual(result, {"id": "parent", "name": "side.co.uk"})
-
-    def test_apex_zone(self):
-        result = RESOLVER.match_zone_for_hostname("side.co.uk", self.zones)
-        self.assertEqual(result["id"], "parent")
-
-    def test_nested_zone_wins(self):
-        result = RESOLVER.match_zone_for_hostname("app.internal.side.co.uk", self.zones)
-        self.assertEqual(result["id"], "nested")
-
-    def test_label_boundary_is_required(self):
-        result = RESOLVER.match_zone_for_hostname("notside.co.uk", self.zones)
-        self.assertIsNone(result)
-
-    def test_wildcard_is_normalized(self):
-        result = RESOLVER.match_zone_for_hostname("*.apps.side.co.uk", self.zones)
-        self.assertEqual(result["id"], "parent")
-
-    def test_case_and_root_dot_are_normalized(self):
-        result = RESOLVER.match_zone_for_hostname("APP.SIDE.CO.UK.", self.zones)
-        self.assertEqual(result["name"], "side.co.uk")
-
-    def test_idna_is_normalized(self):
-        zones = [{"id": "idna", "name": "xn--bcher-kva.example"}]
-        result = RESOLVER.match_zone_for_hostname("shop.bücher.example", zones)
-        self.assertEqual(result["id"], "idna")
-
-    def test_invalid_explicit_name_fails_closed(self):
-        with self.assertRaises(RESOLVER.ZoneResolutionError) as raised:
-            RESOLVER.resolve_zone("app.side.co.uk", self.zones, explicit_zone_name="invalid.example")
-        self.assertEqual(raised.exception.code, "explicit_zone_not_found")
-
-    def test_explicit_zone_must_contain_hostname(self):
-        with self.assertRaises(RESOLVER.ZoneResolutionError) as raised:
-            RESOLVER.resolve_zone("app.side.co.uk", self.zones, explicit_zone_id="primary")
-        self.assertEqual(raised.exception.code, "explicit_zone_conflict")
-
-    def test_unavailable_inventory_can_use_compatibility_default(self):
-        result = RESOLVER.resolve_zone(
-            "app.side.co.uk",
-            [],
-            default_zone_id="parent",
-            inventory_status="unavailable",
-            allow_unverified_default=True,
-        )
-        self.assertEqual(result["source"], "default_unverified")
-        self.assertFalse(result["verified"])
-
-    def test_strict_caller_rejects_unavailable_inventory(self):
-        with self.assertRaises(RESOLVER.ZoneResolutionError) as raised:
-            RESOLVER.resolve_zone("app.side.co.uk", [], inventory_status="unavailable")
-        self.assertEqual(raised.exception.code, "inventory_unavailable")
-
-    def test_default_zone_mismatch_is_rejected(self):
-        with self.assertRaises(RESOLVER.ZoneResolutionError) as raised:
-            RESOLVER.resolve_zone("app.unknown.test", self.zones, default_zone_id="primary")
-        self.assertEqual(raised.exception.code, "default_zone_mismatch")
-
-    def test_missing_default_zone_is_rejected(self):
-        with self.assertRaises(RESOLVER.ZoneResolutionError) as raised:
-            RESOLVER.resolve_zone("app.unknown.test", self.zones, default_zone_id="missing")
-        self.assertEqual(raised.exception.code, "default_zone_not_found")
-
-    def test_invalid_dns_characters_are_rejected(self):
-        with self.assertRaises(RESOLVER.ZoneResolutionError) as raised:
-            RESOLVER.normalize_dns_name("bad_name.side.co.uk")
-        self.assertEqual(raised.exception.code, "hostname_invalid")
-
-    def test_duplicate_normalized_zone_is_rejected(self):
-        zones = [
-            {"id": "one", "name": "side.co.uk"},
-            {"id": "two", "name": "SIDE.CO.UK"},
-        ]
-        with self.assertRaises(RESOLVER.ZoneResolutionError) as raised:
-            RESOLVER.match_zone_for_hostname("app.side.co.uk", zones)
-        self.assertEqual(raised.exception.code, "explicit_zone_conflict")
-
-
-if __name__ == "__main__":
-    unittest.main()

webmail/package.json

@@ -21,17 +21,17 @@
     "@tiptap/extension-underline": "^2.27.2",
     "@tiptap/starter-kit": "^2.6.0",
     "@tiptap/vue-3": "^2.6.0",
-    "axios": "^1.15.0",
+    "axios": "^1.20.0",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.0",
     "date-fns": "^3.6.0",
-    "dompurify": "^3.4.0",
+    "dompurify": "^3.4.14",
     "emoji-mart": "^5.6.0",
     "lucide-vue-next": "^0.400.0",
     "pinia": "^2.2.0",
     "radix-vue": "^1.9.0",
     "tailwind-merge": "^2.5.0",
-    "vue": "^3.5.0",
+    "vue": "^3.5.42",
     "vue-router": "^4.4.0"
   },
   "overrides": {
@@ -42,7 +42,7 @@
   "devDependencies": {
     "@vitejs/plugin-vue": "^5.1.0",
     "autoprefixer": "^10.4.0",
-    "postcss": "^8.4.0",
+    "postcss": "^8.5.26",
     "tailwindcss": "^3.4.0",
     "typescript": "^5.5.0",
     "vite": "^5.4.0",