Apprise Notification support

CHANGELOG.md

@@ -5,9 +5,11 @@ All notable changes to this project will be documented in this file.
 ## [Unreleased]
 
 ### Added
+- **Apprise notifications:** Added opt-in, encrypted notification destinations for rule lifecycle, Cloudflare failures, Docker listener failures, Agent health, and tunnel health. Delivery uses a bounded background queue with cooldowns, startup and intentional-action suppression, redacted settings/status output, and authenticated test delivery. Addresses [#302](https://github.com/ChrispyBacon-dev/DockFlare/issues/302).
 - **Safe Agent decommissioning:** Replaced immediate Agent deletion with a durable prepare, ownership-safe Cloudflare cleanup, and final Agent shutdown workflow. Decommissioned Agents persist a local tombstone, stop their managed tunnel without Docker delete permission, revoke their API key only after final acknowledgement, and cannot silently recreate the tunnel after restart. The Agents UI now shows retryable progress and provides separately acknowledged host cleanup commands that remove the Agent Docker deployment while preserving the shared external `cloudflare-net` network.
 
 ### Fixed
+- **Frontend dependency refresh:** Updated compatible PostCSS, Axios, DOMPurify, Vue, and transitive dependencies.
 - **UI-overridden container lifecycle:** Recreated Docker and Agent-managed containers now reactivate their existing rules, refresh only their runtime container association, and retain UI-controlled routing, origin, tunnel, and Access settings. Fixes [#388](https://github.com/ChrispyBacon-dev/DockFlare/issues/388).
   - Added stable source-rule identity so UI-renamed container-backed rules remain associated with their originating Docker or Agent label binding without creating duplicates.
   - Kept pending rules in tunnel ingress for the full deletion grace period, with generation-aware cleanup, per-tunnel serialization, DNS ownership checks, and durable synchronization retries protecting routes during failures and event races.

dockflare/app/config.py

@@ -157,6 +157,11 @@ DOCKFLARE_PUBLIC_URL = os.getenv('DOCKFLARE_PUBLIC_URL', '')
 
 EMAIL_ENABLED = False
 EMAIL_CONFIG = {}
+NOTIFICATION_CONFIG = {}
+NOTIFICATION_QUEUE_SIZE = _get_int_env('NOTIFICATION_QUEUE_SIZE', 100, minimum=1)
+NOTIFICATION_MAX_BODY_CHARS = _get_int_env('NOTIFICATION_MAX_BODY_CHARS', 4000, minimum=256)
+NOTIFICATION_RESOURCE_LIST_LIMIT = _get_int_env('NOTIFICATION_RESOURCE_LIST_LIMIT', 10, minimum=1)
+NOTIFICATION_HEALTH_INTERVAL_SECONDS = _get_int_env('NOTIFICATION_HEALTH_INTERVAL_SECONDS', 15, minimum=5)
 MAIL_MANAGER_INTERNAL_URL = os.getenv('MAIL_MANAGER_INTERNAL_URL', 'http://dockflare-mail-manager:8025')
 EMAIL_JWT_ALGORITHM = 'EdDSA'
 EMAIL_JWT_ISSUER = 'dockflare-master'

dockflare/app/core/docker_handler.py

@@ -39,6 +39,7 @@ from app.core.cloudflare_api import create_cloudflare_dns_record, get_account_zo
 from app.core.zone_resolver import ZoneResolutionError, normalize_dns_name
 from app.core.access_manager import handle_access_policy_from_labels
 from app.core.utils import get_rule_key, get_source_rule_key, get_label, normalize_access_group_value
+from app.core import notification_manager
 
 def is_valid_hostname(hostname):
     try:
@@ -87,6 +88,8 @@ def process_container_start(container_obj):
 
         container_id_val = None
         container_name_val = "UnknownContainer"
+        activated_resources = []
+        restored_resources = []
         
         try:
             container_id_val = container_obj.id
@@ -298,6 +301,12 @@ def process_container_start(container_obj):
                         if reactivated:
                             mark_rule_tunnel_sync_pending(matched_rule)
                             needs_tunnel_config_update_for_this_container = True
+                            restored_resources.append({
+                                "key": matched_key,
+                                "hostname": matched_rule.get("hostname"),
+                                "path": matched_rule.get("path"),
+                                "source": "docker",
+                            })
                         state_changed_locally_for_this_container |= changed
                         if matched_rule.get("hostname") and matched_rule.get("zone_id"):
                             dns_targets[matched_rule["hostname"]] = {"zone_id": matched_rule["zone_id"]}
@@ -330,6 +339,11 @@ def process_container_start(container_obj):
                     )
                 except ZoneResolutionError as exc:
                     logging.error(f"DOCKER_HANDLER: Zone resolution failed for {rule_key} ({exc.code}). Skipping rule.")
+                    notification_manager.emit(
+                        "cloudflare.dns_failure",
+                        rule_key,
+                        {"hostname": hostname, "path": path_from_item, "operation": "zone resolution", "source": "docker"},
+                    )
                     continue
                 target_zone_id = selected_zone["id"]
                 config_item["zone_name"] = selected_zone.get("name")
@@ -408,6 +422,12 @@ def process_container_start(container_obj):
                             existing_rule["status"] = "active"
                             existing_rule["delete_at"] = None
                             rule_data_changed = True
+                            restored_resources.append({
+                                "key": rule_key,
+                                "hostname": hostname,
+                                "path": path_from_item,
+                                "source": "docker",
+                            })
 
                         tunnel_fields = (
                             "hostname", "path", "service", "zone_id", "no_tls_verify",
@@ -462,6 +482,12 @@ def process_container_start(container_obj):
                         existing_rule = managed_rules[rule_key]
                         state_changed_locally_for_this_container = True
                         needs_tunnel_config_update_for_this_container = True
+                        activated_resources.append({
+                            "key": rule_key,
+                            "hostname": hostname,
+                            "path": path_from_item,
+                            "source": "docker",
+                        })
                         logging.debug(f"DOCKER_HANDLER_NEW_RULE_POST: Added {rule_key}. Rule: {existing_rule}")
 
                     dns_targets[hostname] = {
@@ -478,8 +504,21 @@ def process_container_start(container_obj):
 
             policy_state_changed = False
             for rule_key, policy_payload in policy_jobs:
-                if handle_access_policy_from_labels(rule_key, copy.deepcopy(policy_payload)):
-                    policy_state_changed = True
+                try:
+                    if handle_access_policy_from_labels(rule_key, copy.deepcopy(policy_payload)):
+                        policy_state_changed = True
+                except Exception:
+                    notification_manager.emit(
+                        "cloudflare.access_failure",
+                        rule_key,
+                        {
+                            "hostname": policy_payload.get("hostname"),
+                            "path": policy_payload.get("path"),
+                            "operation": "apply policy",
+                            "source": "docker",
+                        },
+                    )
+                    raise
 
             if policy_state_changed:
                 state_changed_locally_for_this_container = True
@@ -493,6 +532,7 @@ def process_container_start(container_obj):
             if needs_tunnel_config_update_for_this_container:
                 logging.info(f"DOCKER_HANDLER: Triggering tunnel config update for {container_name_val}.")
                 if update_cloudflare_config():
+                    remote_success = True
                     sync_state_changed = False
                     with state_lock:
                         for rule in managed_rules.values():
@@ -510,19 +550,80 @@ def process_container_start(container_obj):
                         for hostname_dns, dns_details in dns_targets.items():
                             target_zone_id_for_dns_item = dns_details.get("zone_id")
                             if target_zone_id_for_dns_item and not hostname_dns.startswith('*.'):
-                                dns_record_id_status = create_cloudflare_dns_record(target_zone_id_for_dns_item, hostname_dns, effective_tunnel_id)
+                                try:
+                                    dns_record_id_status = create_cloudflare_dns_record(
+                                        target_zone_id_for_dns_item, hostname_dns, effective_tunnel_id
+                                    )
+                                except Exception:
+                                    logging.exception("DOCKER_HANDLER: DNS creation raised for %s.", hostname_dns)
+                                    dns_record_id_status = None
                                 if dns_record_id_status and dns_record_id_status not in ["semaphore_timeout", "existing_record_unconfirmed"]:
                                     logging.info(f"DOCKER_HANDLER: DNS for {hostname_dns} in zone {target_zone_id_for_dns_item} OK (ID/Status: {dns_record_id_status}).")
                                 elif not dns_record_id_status:
                                     logging.error(f"DOCKER_HANDLER: CRITICAL - Failed DNS for {hostname_dns} in zone {target_zone_id_for_dns_item}!")
+                                    remote_success = False
+                                    notification_manager.emit(
+                                        "cloudflare.dns_failure",
+                                        f"{target_zone_id_for_dns_item}:{hostname_dns}",
+                                        {"hostname": hostname_dns, "operation": "create", "source": "docker"},
+                                    )
                                     if cloudflared_agent_state:
                                         cloudflared_agent_state["last_action_status"] = f"Error: Failed DNS for {hostname_dns}."
+                                elif dns_record_id_status in ["semaphore_timeout", "existing_record_unconfirmed"]:
+                                    remote_success = False
+                                    notification_manager.emit(
+                                        "cloudflare.dns_failure",
+                                        f"{target_zone_id_for_dns_item}:{hostname_dns}",
+                                        {"hostname": hostname_dns, "operation": "create", "source": "docker"},
+                                    )
                             elif not target_zone_id_for_dns_item:
                                 logging.error(f"DOCKER_HANDLER: No Zone ID for DNS for {hostname_dns} - cannot manage record.")
+                                remote_success = False
+                                notification_manager.emit(
+                                    "cloudflare.dns_failure",
+                                    hostname_dns,
+                                    {"hostname": hostname_dns, "operation": "create", "source": "docker"},
+                                )
                     else:
                         logging.error(f"DOCKER_HANDLER: Missing effective Tunnel ID - cannot manage DNS records for {container_name_val}.")
+                        remote_success = False
+                        notification_manager.emit(
+                            "cloudflare.tunnel_failure",
+                            container_id_val,
+                            {"container_name": container_name_val, "container_id": container_id_val[:12], "operation": "resolve tunnel", "source": "docker"},
+                        )
+                    if remote_success:
+                        common_context = {
+                            "container_name": container_name_val,
+                            "container_id": container_id_val[:12],
+                            "source": "docker",
+                            "public_url": config.DOCKFLARE_PUBLIC_URL,
+                        }
+                        if activated_resources:
+                            notification_manager.emit(
+                                "rule.activated",
+                                container_id_val,
+                                {**common_context, "resources": activated_resources},
+                            )
+                        if restored_resources:
+                            notification_manager.emit(
+                                "rule.restored",
+                                container_id_val,
+                                {**common_context, "resources": restored_resources},
+                            )
                 else:
                     logging.error(f"DOCKER_HANDLER: Failed to update Cloudflare tunnel config for {container_name_val}. DNS records not managed.")
+                    notification_manager.emit(
+                        "cloudflare.tunnel_failure",
+                        str(tunnel_state.get("id") or config.EXTERNAL_TUNNEL_ID or container_id_val),
+                        {
+                            "container_name": container_name_val,
+                            "container_id": container_id_val[:12],
+                            "tunnel_id": str(tunnel_state.get("id") or config.EXTERNAL_TUNNEL_ID or "")[:12],
+                            "operation": "update",
+                            "source": "docker",
+                        },
+                    )
 
 
         except NotFound:
@@ -543,6 +644,8 @@ def schedule_container_stop(container_id_val):
         logging.info(f"Processing stop event for container {container_id_val[:12]}.")
         
         state_changed_after_stop_processing = False
+        pending_resources = []
+        delete_at_value = None
         with state_lock:
             rule_keys_affected_by_stop = []
             for r_key, details in managed_rules.items():
@@ -561,6 +664,13 @@ def schedule_container_stop(container_id_val):
                         rule["delete_at"] = datetime.now(timezone.utc) + grace_delta
                         rule["lifecycle_generation"] = int(rule.get("lifecycle_generation") or 0) + 1
                         logging.info(f"Rule for {rule_key_to_schedule} (from stopped container {container_id_val[:12]}) scheduled for deletion at {rule['delete_at'].isoformat()}")
+                        delete_at_value = rule['delete_at'].isoformat()
+                        pending_resources.append({
+                            "key": rule_key_to_schedule,
+                            "hostname": rule.get("hostname"),
+                            "path": rule.get("path"),
+                            "source": "docker",
+                        })
                         state_changed_after_stop_processing = True
                     else:
                         logging.info(f"Rule for {rule_key_to_schedule} from stopped container {container_id_val[:12]} was already pending deletion.")
@@ -570,6 +680,19 @@ def schedule_container_stop(container_id_val):
             if state_changed_after_stop_processing:
                 save_state()
                 publish_state_event('snapshot_refresh')
+        if pending_resources:
+            notification_manager.emit(
+                "rule.pending_deletion",
+                container_id_val,
+                {
+                    "container_id": container_id_val[:12],
+                    "source": "docker",
+                    "delete_at": delete_at_value,
+                    "grace_period_seconds": current_app.config.get('GRACE_PERIOD_SECONDS', 28800),
+                    "resources": pending_resources,
+                    "public_url": config.DOCKFLARE_PUBLIC_URL,
+                },
+            )
 
 def docker_event_listener(stop_event_param, label_prefix):
     if not docker_client:
@@ -646,6 +769,11 @@ def docker_event_listener(stop_event_param, label_prefix):
 
     if error_count >= max_errors:
         logging.error(f"Docker event listener for {label_prefix} stopping after multiple consecutive errors.")
+        notification_manager.emit(
+            "docker.listener_failure",
+            label_prefix,
+            {"message": "The Docker event listener stopped after repeated connection failures."},
+        )
     logging.info(f"Docker event listener for {label_prefix} stopped.")
 
 def start_event_listeners(stop_event):

dockflare/app/core/notification_manager.py

@@ -0,0 +1,680 @@
+import copy
+import itertools
+import logging
+import os
+import queue
+import threading
+import time
+import uuid
+from datetime import datetime, timezone
+from urllib.parse import urlsplit, urlunsplit
+
+
+EVENT_DEFINITIONS = {
+    "rule.activated": ("rule_activated", "success", False),
+    "rule.restored": ("rule_restored", "success", False),
+    "rule.pending_deletion": ("rule_pending_deletion", "warning", False),
+    "rule.deleted": ("rule_deleted", "info", True),
+    "cloudflare.tunnel_failure": ("cloudflare_tunnel_failure", "failure", True),
+    "cloudflare.dns_failure": ("cloudflare_dns_failure", "failure", True),
+    "cloudflare.access_failure": ("cloudflare_access_failure", "failure", True),
+    "docker.listener_failure": ("docker_listener_failure", "failure", True),
+    "agent.offline": ("agent_offline", "failure", True),
+    "agent.online": ("agent_online", "success", True),
+    "tunnel.down": ("tunnel_down", "failure", True),
+    "tunnel.recovered": ("tunnel_recovered", "success", True),
+}
+
+DEFAULT_NOTIFICATION_CONFIG = {
+    "enabled": False,
+    "urls": [],
+    "events": {definition[0]: definition[2] for definition in EVENT_DEFINITIONS.values()},
+    "failure_cooldown_seconds": 900,
+}
+
+BOOTSTRAP_SUPPRESSED_EVENTS = {
+    "rule.activated",
+    "rule.restored",
+    "rule.pending_deletion",
+    "rule.deleted",
+    "agent.online",
+    "tunnel.recovered",
+}
+
+COOLDOWN_EVENTS = {
+    "cloudflare.tunnel_failure",
+    "cloudflare.dns_failure",
+    "cloudflare.access_failure",
+    "docker.listener_failure",
+    "agent.offline",
+    "tunnel.down",
+}
+
+RECOVERY_PAIRS = {
+    "agent.online": "agent.offline",
+    "tunnel.recovered": "tunnel.down",
+}
+
+TITLE_MAP = {
+    "rule.activated": "DockFlare: rule activated",
+    "rule.restored": "DockFlare: rule restored",
+    "rule.pending_deletion": "DockFlare: rule scheduled for deletion",
+    "rule.deleted": "DockFlare: rule deleted",
+    "cloudflare.tunnel_failure": "DockFlare: tunnel update failed",
+    "cloudflare.dns_failure": "DockFlare: DNS operation failed",
+    "cloudflare.access_failure": "DockFlare: Access operation failed",
+    "docker.listener_failure": "DockFlare: Docker listener stopped",
+    "agent.offline": "DockFlare: Agent offline",
+    "agent.online": "DockFlare: Agent recovered",
+    "tunnel.down": "DockFlare: tunnel down",
+    "tunnel.recovered": "DockFlare: tunnel recovered",
+    "notification.test": "DockFlare: test notification",
+}
+
+
+def normalize_notification_config(value):
+    normalized = copy.deepcopy(DEFAULT_NOTIFICATION_CONFIG)
+    if not isinstance(value, dict):
+        return normalized
+
+    normalized["enabled"] = bool(value.get("enabled", False))
+    urls = value.get("urls", [])
+    if isinstance(urls, list):
+        seen = set()
+        valid_shape = []
+        for item in urls:
+            if not isinstance(item, str):
+                continue
+            item = item.strip()
+            if not item or len(item.encode("utf-8")) > 4096 or item in seen:
+                continue
+            seen.add(item)
+            valid_shape.append(item)
+            if len(valid_shape) == 32:
+                break
+        normalized["urls"] = valid_shape
+
+    supplied_events = value.get("events", {})
+    if isinstance(supplied_events, dict):
+        for event_key in normalized["events"]:
+            if event_key in supplied_events:
+                normalized["events"][event_key] = bool(supplied_events[event_key])
+
+    cooldown = value.get("failure_cooldown_seconds", 900)
+    if isinstance(cooldown, int) and not isinstance(cooldown, bool) and 60 <= cooldown <= 86400:
+        normalized["failure_cooldown_seconds"] = cooldown
+    return normalized
+
+
+def sanitize_service(value):
+    if not isinstance(value, str) or not value.strip():
+        return None
+    value = value.strip()
+    try:
+        parsed = urlsplit(value)
+    except ValueError:
+        return None
+    if not parsed.scheme:
+        return value[:256] if all(char not in value for char in ("?", "#", "@")) else None
+    try:
+        hostname = parsed.hostname
+        port_value = parsed.port
+    except ValueError:
+        return parsed.scheme[:32] if parsed.scheme else None
+    if not hostname:
+        return parsed.scheme
+    port = f":{port_value}" if port_value else ""
+    return urlunsplit((parsed.scheme, f"{hostname}{port}", parsed.path or "", "", ""))[:256]
+
+
+def redact_destination(url):
+    if not isinstance(url, str):
+        return "configured destination"
+    try:
+        scheme = urlsplit(url.strip()).scheme.lower()
+    except ValueError:
+        scheme = ""
+    if not scheme or not scheme.replace("+", "").replace("-", "").isalnum():
+        return "configured destination"
+    return f"{scheme}://configured destination"
+
+
+class NotificationManager:
+    def __init__(self, queue_size=100, max_body_chars=4000, resource_limit=10):
+        self._queue_size = max(1, int(queue_size))
+        self._max_body_chars = max(256, int(max_body_chars))
+        self._resource_limit = max(1, int(resource_limit))
+        self._queue = queue.PriorityQueue(maxsize=self._queue_size)
+        self._sequence = itertools.count()
+        self._lock = threading.RLock()
+        self._config = normalize_notification_config(None)
+        self._apprise = None
+        self._available = False
+        self._invalid_destination_count = 0
+        self._stop_event = None
+        self._worker = None
+        self._accepting = True
+        self._bootstrap = True
+        self._cooldowns = {}
+        self._outages_emitted = set()
+        self._test_jobs = {}
+        self._dropped_events = 0
+        self._last_drop_log_at = 0.0
+        self._last_attempt_at = None
+        self._last_success_at = None
+        self._last_failure_at = None
+        self._agent_health = {}
+        self._tunnel_health = {}
+        self._intentional_tunnels = {}
+
+    @staticmethod
+    def _load_apprise():
+        import apprise
+        return apprise
+
+    def _build_apprise(self, urls, reject_invalid=False):
+        try:
+            apprise_module = self._load_apprise()
+            asset = apprise_module.AppriseAsset(
+                app_id="DockFlare",
+                app_desc="DockFlare Notifications",
+                app_url="https://dockflare.app",
+                secure_logging=True,
+            )
+            instance = apprise_module.Apprise(asset=asset)
+        except Exception as exc:
+            logging.error("NOTIFY_CONFIG: Apprise unavailable (%s).", type(exc).__name__)
+            return None, [(0, "unavailable")]
+
+        invalid = []
+        for index, url in enumerate(urls, 1):
+            try:
+                if not instance.add(url):
+                    invalid.append((index, self._safe_scheme(url)))
+            except Exception:
+                invalid.append((index, self._safe_scheme(url)))
+        if reject_invalid and invalid:
+            return None, invalid
+        return instance if len(instance) else None, invalid
+
+    @staticmethod
+    def _safe_scheme(url):
+        try:
+            scheme = urlsplit(url).scheme.lower()
+        except (TypeError, ValueError):
+            return "unknown"
+        return scheme if scheme and scheme.replace("+", "").replace("-", "").isalnum() else "unknown"
+
+    def validate_urls(self, urls):
+        if not isinstance(urls, list) or len(urls) > 32:
+            return False, [(0, "invalid")]
+        for index, url in enumerate(urls, 1):
+            if not isinstance(url, str) or not url.strip() or len(url.strip().encode("utf-8")) > 4096:
+                return False, [(index, self._safe_scheme(url))]
+        normalized = normalize_notification_config({"urls": urls})["urls"]
+        _instance, invalid = self._build_apprise(normalized, reject_invalid=True)
+        return not invalid, invalid
+
+    def configure(self, value):
+        normalized = normalize_notification_config(value)
+        instance = None
+        invalid = []
+        if normalized["urls"]:
+            instance, invalid = self._build_apprise(normalized["urls"])
+        with self._lock:
+            self._config = normalized
+            self._apprise = instance
+            self._invalid_destination_count = len(invalid)
+            self._available = instance is not None and len(instance) > 0
+        if invalid:
+            logging.warning("NOTIFY_CONFIG: Skipped %s invalid destination(s).", len(invalid))
+        logging.info(
+            "NOTIFY_CONFIG: enabled=%s configured=%s valid=%s",
+            normalized["enabled"],
+            len(normalized["urls"]),
+            len(instance) if instance is not None else 0,
+        )
+        return self.get_public_status()
+
+    def start(self, stop_event):
+        with self._lock:
+            if self._worker and self._worker.is_alive():
+                return self._worker
+            self._stop_event = stop_event
+            self._accepting = True
+            self._worker = threading.Thread(target=self._run, name="NotificationWorker", daemon=True)
+            self._worker.start()
+            logging.info("NOTIFY_WORKER: Started.")
+            return self._worker
+
+    def stop(self, timeout_seconds=5):
+        with self._lock:
+            self._accepting = False
+            worker = self._worker
+        try:
+            self._queue.put_nowait((-1, next(self._sequence), None))
+        except queue.Full:
+            pass
+        if worker and worker.is_alive():
+            worker.join(timeout=max(0, timeout_seconds))
+
+    def begin_bootstrap(self):
+        with self._lock:
+            self._bootstrap = True
+
+    def end_bootstrap(self):
+        with self._lock:
+            self._bootstrap = False
+
+    def emit(self, event_type, resource_id, context=None, notify_type=None):
+        return self._enqueue(event_type, resource_id, context, notify_type, False, None, 10)
+
+    def send_test(self):
+        job_id = str(uuid.uuid4())
+        with self._lock:
+            self._expire_test_jobs_locked()
+            self._test_jobs[job_id] = {
+                "status": "pending",
+                "created_at": time.time(),
+                "completed_at": None,
+            }
+        accepted = self._enqueue(
+            "notification.test",
+            "test",
+            {"message": "DockFlare notification delivery is configured correctly."},
+            "info",
+            True,
+            job_id,
+            0,
+        )
+        if not accepted:
+            with self._lock:
+                self._test_jobs[job_id].update({"status": "failure", "completed_at": self._utc_now()})
+        return job_id, accepted
+
+    def get_test_status(self, job_id):
+        with self._lock:
+            self._expire_test_jobs_locked()
+            value = self._test_jobs.get(job_id)
+            return copy.deepcopy(value) if value else None
+
+    def _expire_test_jobs_locked(self):
+        cutoff = time.time() - 600
+        for job_id in [key for key, value in self._test_jobs.items() if value["created_at"] < cutoff]:
+            self._test_jobs.pop(job_id, None)
+
+    def _enqueue(self, event_type, resource_id, context, notify_type, is_test, job_id, priority):
+        if event_type != "notification.test" and event_type not in EVENT_DEFINITIONS:
+            return False
+        safe_resource = str(resource_id or "unknown")[:512]
+        now = time.monotonic()
+        cooldown_key = None
+        outage_key = None
+        with self._lock:
+            config_snapshot = copy.deepcopy(self._config)
+            if not self._accepting or not config_snapshot["enabled"] or not self._available:
+                return False
+            if not is_test:
+                setting_key, default_type, _default_enabled = EVENT_DEFINITIONS[event_type]
+                if not config_snapshot["events"].get(setting_key, False):
+                    return False
+                if self._bootstrap and event_type in BOOTSTRAP_SUPPRESSED_EVENTS:
+                    logging.info("NOTIFY_SUPPRESSED: %s during bootstrap.", event_type)
+                    return False
+                notify_type = notify_type or default_type
+                cooldown_key = f"{event_type}:{safe_resource}"
+                if event_type in COOLDOWN_EVENTS:
+                    last = self._cooldowns.get(cooldown_key)
+                    if last is not None and now - last < config_snapshot["failure_cooldown_seconds"]:
+                        logging.info("NOTIFY_SUPPRESSED: %s cooldown active.", event_type)
+                        return False
+                recovery_for = RECOVERY_PAIRS.get(event_type)
+                if recovery_for:
+                    outage_key = f"{recovery_for}:{safe_resource}"
+                    if outage_key not in self._outages_emitted:
+                        return False
+
+        event = {
+            "event_id": str(uuid.uuid4()),
+            "event_type": event_type,
+            "resource_id": safe_resource,
+            "occurred_at": self._utc_now(),
+            "notify_type": notify_type or "info",
+            "context": self._sanitize_context(context or {}),
+            "is_test": is_test,
+            "job_id": job_id,
+        }
+        try:
+            self._queue.put_nowait((priority, next(self._sequence), event))
+            with self._lock:
+                if cooldown_key and event_type in COOLDOWN_EVENTS:
+                    self._cooldowns[cooldown_key] = now
+                    self._outages_emitted.add(cooldown_key)
+                if outage_key:
+                    self._outages_emitted.discard(outage_key)
+                    self._cooldowns.pop(outage_key, None)
+            logging.info("NOTIFY_QUEUED: %s event_id=%s", event_type, event["event_id"])
+            return True
+        except queue.Full:
+            with self._lock:
+                self._dropped_events += 1
+                if now - self._last_drop_log_at >= 60:
+                    logging.warning("NOTIFY_DROPPED: Notification queue is full.")
+                    self._last_drop_log_at = now
+            return False
+
+    def _run(self):
+        while True:
+            stop_requested = self._stop_event is not None and self._stop_event.is_set()
+            if stop_requested and self._queue.empty():
+                break
+            try:
+                _priority, _sequence, event = self._queue.get(timeout=0.5)
+            except queue.Empty:
+                continue
+            if event is None:
+                self._queue.task_done()
+                if stop_requested or not self._accepting:
+                    break
+                continue
+            try:
+                self._deliver(event)
+            except Exception as exc:
+                logging.error(
+                    "NOTIFY_FAILED: event_type=%s event_id=%s error=%s",
+                    event["event_type"], event["event_id"], type(exc).__name__,
+                )
+                self._record_delivery(event, False)
+            finally:
+                self._queue.task_done()
+        logging.info("NOTIFY_WORKER: Stopped.")
+
+    def _deliver(self, event):
+        with self._lock:
+            instance = self._apprise
+        if instance is None:
+            self._record_delivery(event, False)
+            return
+        title, body = self._render(event)
+        notify_type = self._resolve_notify_type(event["notify_type"])
+        started = time.monotonic()
+        success = bool(instance.notify(title=title, body=body, notify_type=notify_type))
+        duration = time.monotonic() - started
+        self._record_delivery(event, success)
+        log_method = logging.info if success else logging.error
+        log_method(
+            "%s: event_type=%s event_id=%s destinations=%s duration=%.2fs",
+            "NOTIFY_DELIVERED" if success else "NOTIFY_FAILED",
+            event["event_type"],
+            event["event_id"],
+            len(instance),
+            duration,
+        )
+
+    def _resolve_notify_type(self, value):
+        try:
+            apprise_module = self._load_apprise()
+            return {
+                "info": apprise_module.NotifyType.INFO,
+                "success": apprise_module.NotifyType.SUCCESS,
+                "warning": apprise_module.NotifyType.WARNING,
+                "failure": apprise_module.NotifyType.FAILURE,
+            }.get(value, apprise_module.NotifyType.INFO)
+        except Exception:
+            return value
+
+    def _record_delivery(self, event, success):
+        completed_at = self._utc_now()
+        with self._lock:
+            self._last_attempt_at = completed_at
+            if success:
+                self._last_success_at = completed_at
+            else:
+                self._last_failure_at = completed_at
+            job_id = event.get("job_id")
+            if job_id and job_id in self._test_jobs:
+                self._test_jobs[job_id].update({
+                    "status": "success" if success else "failure",
+                    "completed_at": completed_at,
+                })
+
+    def _render(self, event):
+        event_type = event["event_type"]
+        context = event["context"]
+        title = TITLE_MAP.get(event_type, "DockFlare notification")
+        resources = context.get("resources") if isinstance(context.get("resources"), list) else []
+        if len(resources) > 1 and event_type.startswith("rule."):
+            title = title.replace("rule ", f"{len(resources)} rules ")
+
+        lines = []
+        message = context.get("message")
+        if message:
+            lines.append(str(message))
+        scalar_fields = (
+            ("operation", "Operation"),
+            ("hostname", "Hostname"),
+            ("path", "Path"),
+            ("source", "Source"),
+            ("container_name", "Container"),
+            ("container_id", "Container ID"),
+            ("agent_name", "Agent"),
+            ("agent_id", "Agent ID"),
+            ("tunnel_name", "Tunnel"),
+            ("tunnel_id", "Tunnel ID"),
+            ("delete_at", "Deletion deadline"),
+            ("grace_period_seconds", "Grace period (seconds)"),
+            ("retry_count", "Retry count"),
+        )
+        for key, label in scalar_fields:
+            if context.get(key) not in (None, ""):
+                lines.append(f"{label}: {context[key]}")
+        if context.get("service"):
+            service = sanitize_service(context["service"])
+            if service:
+                lines.append(f"Service: {service}")
+
+        if resources:
+            lines.append("Affected resources:")
+            for resource in resources[:self._resource_limit]:
+                if not isinstance(resource, dict):
+                    continue
+                hostname = str(resource.get("hostname") or resource.get("key") or "resource")[:255]
+                path = str(resource.get("path") or "")[:255]
+                lines.append(f"- {hostname}{path}")
+            omitted = len(resources) - self._resource_limit
+            if omitted > 0:
+                lines.append(f"- ... and {omitted} more")
+        if context.get("public_url"):
+            public_url = sanitize_service(context["public_url"])
+            if public_url:
+                lines.append(f"DockFlare: {public_url}")
+        if not lines:
+            lines.append(f"Event: {event_type}")
+        body = "\n".join(lines)
+        if len(body) > self._max_body_chars:
+            body = body[: self._max_body_chars - 16].rstrip() + "\n[truncated]"
+        return title, body
+
+    def _sanitize_context(self, context):
+        allowed = {
+            "message", "operation", "hostname", "path", "service", "source",
+            "container_name", "container_id", "agent_name", "agent_id",
+            "tunnel_name", "tunnel_id", "delete_at", "grace_period_seconds",
+            "retry_count", "resources", "public_url",
+        }
+        sanitized = {}
+        for key in allowed:
+            value = context.get(key)
+            if value is None:
+                continue
+            if key == "resources" and isinstance(value, list):
+                sanitized[key] = [
+                    {
+                        field: str(item[field])[:512]
+                        for field in ("key", "hostname", "path", "source")
+                        if isinstance(item, dict) and field in item and item[field] is not None
+                    }
+                    for item in value[:1000]
+                    if isinstance(item, dict)
+                ]
+            elif isinstance(value, (str, int, float, bool)):
+                sanitized[key] = str(value)[:2048]
+        return sanitized
+
+    def get_public_status(self):
+        with self._lock:
+            self._expire_test_jobs_locked()
+            urls = list(self._config["urls"])
+            return {
+                "enabled": self._config["enabled"],
+                "available": self._available,
+                "configured_destination_count": len(urls),
+                "valid_destination_count": max(0, len(urls) - self._invalid_destination_count),
+                "destinations": [
+                    {"index": index, "scheme": self._safe_scheme(url), "summary": redact_destination(url)}
+                    for index, url in enumerate(urls, 1)
+                ],
+                "events": copy.deepcopy(self._config["events"]),
+                "failure_cooldown_seconds": self._config["failure_cooldown_seconds"],
+                "queue_depth": self._queue.qsize(),
+                "queue_capacity": self._queue_size,
+                "dropped_events": self._dropped_events,
+                "last_attempt_at": self._last_attempt_at,
+                "last_success_at": self._last_success_at,
+                "last_failure_at": self._last_failure_at,
+                "bootstrap": self._bootstrap,
+            }
+
+    def check_agent_health(self, agents_snapshot, heartbeat_timeout, now=None, decommissioning_ids=None):
+        now = now or datetime.now(timezone.utc)
+        decommissioning_ids = set(decommissioning_ids or [])
+        for agent_id, agent in (agents_snapshot or {}).items():
+            if not isinstance(agent, dict) or agent.get("status") != "enrolled" or agent_id in decommissioning_ids:
+                continue
+            online = False
+            try:
+                last_seen = datetime.fromisoformat(str(agent.get("last_seen", "")).replace("Z", "+00:00"))
+                if last_seen.tzinfo is None:
+                    last_seen = last_seen.replace(tzinfo=timezone.utc)
+                online = (now - last_seen.astimezone(timezone.utc)).total_seconds() <= heartbeat_timeout
+            except (TypeError, ValueError):
+                online = False
+            previous = self._agent_health.get(agent_id)
+            self._agent_health[agent_id] = online
+            if previous is None:
+                continue
+            context = {
+                "agent_name": agent.get("display_name") or f"agent-{agent_id[:8]}",
+                "agent_id": agent_id[:12],
+            }
+            if previous and not online:
+                self.emit("agent.offline", agent_id, context)
+            elif not previous and online:
+                self.emit("agent.online", agent_id, context)
+
+    def check_tunnel_health(self, tunnel_id, status, context=None, intentional=False):
+        if not tunnel_id:
+            return
+        now = time.monotonic()
+        with self._lock:
+            intentional_until = self._intentional_tunnels.get(tunnel_id, 0)
+            if intentional_until and intentional_until <= now:
+                self._intentional_tunnels.pop(tunnel_id, None)
+                intentional_until = 0
+        intentional = intentional or intentional_until > now
+        healthy = str(status or "").lower() == "running"
+        state = self._tunnel_health.setdefault(tunnel_id, {"healthy": None, "unhealthy_checks": 0})
+        previous = state["healthy"]
+        if healthy:
+            state["healthy"] = True
+            state["unhealthy_checks"] = 0
+            if previous is False:
+                self.emit("tunnel.recovered", tunnel_id, context or {})
+            return
+        if intentional:
+            state["healthy"] = False
+            state["unhealthy_checks"] = 0
+            return
+        state["unhealthy_checks"] += 1
+        if previous is None:
+            state["healthy"] = False
+            return
+        if previous is True and state["unhealthy_checks"] >= 2:
+            state["healthy"] = False
+            self.emit("tunnel.down", tunnel_id, context or {})
+
+    def suppress_tunnel_health(self, tunnel_id, seconds=120):
+        if not tunnel_id:
+            return
+        with self._lock:
+            self._intentional_tunnels[tunnel_id] = time.monotonic() + max(1, int(seconds))
+
+    @staticmethod
+    def _utc_now():
+        return datetime.now(timezone.utc).isoformat()
+
+
+def _runtime_int(name, default, minimum):
+    try:
+        return max(minimum, int(os.getenv(name, str(default))))
+    except (TypeError, ValueError):
+        return default
+
+
+notification_manager = NotificationManager(
+    queue_size=_runtime_int("NOTIFICATION_QUEUE_SIZE", 100, 1),
+    max_body_chars=_runtime_int("NOTIFICATION_MAX_BODY_CHARS", 4000, 256),
+    resource_limit=_runtime_int("NOTIFICATION_RESOURCE_LIST_LIMIT", 10, 1),
+)
+
+
+def configure(value):
+    return notification_manager.configure(value)
+
+
+def start(stop_event):
+    return notification_manager.start(stop_event)
+
+
+def stop(timeout_seconds=5):
+    return notification_manager.stop(timeout_seconds)
+
+
+def begin_bootstrap():
+    return notification_manager.begin_bootstrap()
+
+
+def end_bootstrap():
+    return notification_manager.end_bootstrap()
+
+
+def emit(event_type, resource_id, context=None, notify_type=None):
+    return notification_manager.emit(event_type, resource_id, context, notify_type)
+
+
+def send_test():
+    return notification_manager.send_test()
+
+
+def get_test_status(job_id):
+    return notification_manager.get_test_status(job_id)
+
+
+def get_public_status():
+    return notification_manager.get_public_status()
+
+
+def validate_urls(urls):
+    return notification_manager.validate_urls(urls)
+
+
+def check_agent_health(agents_snapshot, heartbeat_timeout, now=None, decommissioning_ids=None):
+    return notification_manager.check_agent_health(agents_snapshot, heartbeat_timeout, now, decommissioning_ids)
+
+
+def check_tunnel_health(tunnel_id, status, context=None, intentional=False):
+    return notification_manager.check_tunnel_health(tunnel_id, status, context, intentional)
+
+
+def suppress_tunnel_health(tunnel_id, seconds=120):
+    return notification_manager.suppress_tunnel_health(tunnel_id, seconds)

dockflare/app/core/reconciler.py

@@ -47,6 +47,7 @@ from app.core.access_manager import (
 )
 from app.core.tunnel_manager import get_tunnel_operation_lock, update_cloudflare_config
 from app.core.utils import get_rule_key, get_source_rule_key, get_label
+from app.core import notification_manager
 
 def _get_hostname_configs_from_container(container_obj):
     labels = container_obj.labels
@@ -780,6 +781,11 @@ def _run_reconciliation_logic():
             if not update_cloudflare_config(effective_tunnel_id):
                 logging.error("[Reconcile] Failed to update Cloudflare tunnel configuration.")
                 current_app.reconciliation_info["status"] = "Error: Failed tunnel config update."
+                notification_manager.emit(
+                    "cloudflare.tunnel_failure",
+                    str(effective_tunnel_id or "master"),
+                    {"tunnel_id": str(effective_tunnel_id or "")[:12], "operation": "reconciliation update"},
+                )
             else:
                 logging.info("[Reconcile] Cloudflare tunnel configuration updated successfully.")
                 current_app.reconciliation_info["status"] = "Tunnel configuration updated."
@@ -809,11 +815,26 @@ def _run_reconciliation_logic():
                     break
                 
                 if tunnel_id_dns and not hostname_dns.startswith('*.'):
-                    create_cloudflare_dns_record(zone_id_dns, hostname_dns, tunnel_id_dns)
+                    try:
+                        dns_result = create_cloudflare_dns_record(zone_id_dns, hostname_dns, tunnel_id_dns)
+                    except Exception:
+                        logging.exception("[Reconcile] DNS creation raised for %s.", hostname_dns)
+                        dns_result = None
+                    if not dns_result or dns_result in {"semaphore_timeout", "existing_record_unconfirmed"}:
+                        notification_manager.emit(
+                            "cloudflare.dns_failure",
+                            f"{zone_id_dns}:{hostname_dns}",
+                            {"hostname": hostname_dns, "operation": "reconciliation create"},
+                        )
                     if config.USE_EXTERNAL_CLOUDFLARED:
                         time.sleep(0.1)
                 elif not tunnel_id_dns:
                     logging.error(f"[Reconcile] Cannot setup DNS for {hostname_dns}: Tunnel ID is missing.")
+                    notification_manager.emit(
+                        "cloudflare.tunnel_failure",
+                        hostname_dns,
+                        {"hostname": hostname_dns, "operation": "reconciliation resolve tunnel"},
+                    )
 
         for old_hostname, old_zone_id, old_tunnel_id in dns_cleanup_after_sync:
             with state_lock:
@@ -830,7 +851,12 @@ def _run_reconciliation_logic():
                     for rule in managed_rules.values()
                 )
             if not sync_pending and not still_owned and not old_hostname.startswith("*."):
-                delete_cloudflare_dns_record(old_zone_id, old_hostname, old_tunnel_id)
+                if not delete_cloudflare_dns_record(old_zone_id, old_hostname, old_tunnel_id):
+                    notification_manager.emit(
+                        "cloudflare.dns_failure",
+                        f"{old_zone_id}:{old_hostname}",
+                        {"hostname": old_hostname, "operation": "reconciliation delete"},
+                    )
                 
         current_app.reconciliation_info["in_progress"] = False
         current_app.reconciliation_info["progress"] = 100 
@@ -921,6 +947,10 @@ def cleanup_expired_rules_once(now=None):
     for candidate in candidates:
         groups.setdefault(candidate["effective_tunnel_id"], []).append(candidate)
     committed = 0
+    committed_candidates = []
+    tunnel_failures = []
+    dns_failures = []
+    access_failures = []
     for tunnel_id in sorted(groups, key=lambda value: str(value or "")):
         if not tunnel_id:
             logging.error("Cleanup retained %s rules because their effective tunnel is missing.", len(groups[tunnel_id]))
@@ -928,7 +958,10 @@ def cleanup_expired_rules_once(now=None):
         with get_tunnel_operation_lock(tunnel_id):
             with state_lock:
                 valid = [candidate for candidate in groups[tunnel_id] if _candidate_still_matches(candidate)]
-            if not valid or not update_cloudflare_config(tunnel_id):
+            if not valid:
+                continue
+            if not update_cloudflare_config(tunnel_id):
+                tunnel_failures.append(tunnel_id)
                 continue
             successful = []
             for candidate in valid:
@@ -947,8 +980,10 @@ def cleanup_expired_rules_once(now=None):
                         for key, rule in managed_rules.items()
                     )
                 if candidate["hostname"] and not dns_shared and not delete_cloudflare_dns_record(candidate["zone_id"], candidate["hostname"], tunnel_id):
+                    dns_failures.append(candidate)
                     continue
                 if candidate["access_app_id"] and not app_shared and not delete_cloudflare_access_application(candidate["access_app_id"]):
+                    access_failures.append(candidate)
                     continue
                 successful.append(candidate)
             with state_lock:
@@ -956,10 +991,45 @@ def cleanup_expired_rules_once(now=None):
                     if _candidate_still_matches(candidate):
                         del managed_rules[candidate["key"]]
                         committed += 1
+                        committed_candidates.append(candidate)
                 if successful:
                     save_state()
     if committed:
         publish_state_event("snapshot_refresh")
+        notification_manager.emit(
+            "rule.deleted",
+            f"cleanup:{int(current_time.timestamp())}",
+            {
+                "source": "cleanup",
+                "resources": [
+                    {
+                        "key": candidate["key"],
+                        "hostname": candidate.get("hostname"),
+                        "source": candidate.get("source"),
+                    }
+                    for candidate in committed_candidates
+                ],
+                "public_url": config.DOCKFLARE_PUBLIC_URL,
+            },
+        )
+    for tunnel_id in tunnel_failures:
+        notification_manager.emit(
+            "cloudflare.tunnel_failure",
+            tunnel_id,
+            {"tunnel_id": str(tunnel_id)[:12], "operation": "cleanup update"},
+        )
+    for candidate in dns_failures:
+        notification_manager.emit(
+            "cloudflare.dns_failure",
+            f"{candidate.get('zone_id')}:{candidate.get('hostname')}",
+            {"hostname": candidate.get("hostname"), "operation": "delete", "source": candidate.get("source")},
+        )
+    for candidate in access_failures:
+        notification_manager.emit(
+            "cloudflare.access_failure",
+            candidate["key"],
+            {"hostname": candidate.get("hostname"), "operation": "delete", "source": candidate.get("source")},
+        )
     return committed
 
 
@@ -990,6 +1060,16 @@ def retry_pending_tunnel_sync(now=None, force=False):
                 rule["tunnel_sync_attempts"] = int(rule.get("tunnel_sync_attempts") or 0) + 1
                 changed = True
         success = update_cloudflare_config(tunnel_id)
+        if not success:
+            notification_manager.emit(
+                "cloudflare.tunnel_failure",
+                tunnel_id,
+                {
+                    "tunnel_id": str(tunnel_id)[:12],
+                    "operation": "retry pending synchronization",
+                    "retry_count": max((int(rule.get("tunnel_sync_attempts") or 0) for rule in rules), default=0),
+                },
+            )
         if success:
             with state_lock:
                 for rule in managed_rules.values():

dockflare/app/i18n/en.json

@@ -164,6 +164,38 @@
     "settings.backup_restore": "Backup & Restore",
     "settings.security": "Security",
     "settings.oauth_authentication": "OAuth Authentication",
+    "settings.notifications": "Notifications",
+    "settings.notifications_description": "Send selected DockFlare lifecycle and failure alerts through one or more Apprise destinations.",
+    "settings.notification_status": "Status",
+    "settings.notification_unavailable": "Unavailable",
+    "settings.notification_disabled": "Disabled",
+    "settings.configured_destinations": "Configured destinations",
+    "settings.enable_notifications": "Enable notifications",
+    "settings.replacement_apprise_urls": "Replacement Apprise URLs",
+    "settings.replacement_apprise_urls_help": "Enter one URL per line. Leave blank to keep existing destinations. Saved URLs are never shown again.",
+    "settings.clear_notification_destinations": "Clear all configured destinations",
+    "settings.notification_cooldown": "Failure cooldown (seconds)",
+    "settings.notification_cooldown_help": "Suppresses repeated alerts for the same incident for 60 to 86400 seconds.",
+    "settings.notification_events": "Events",
+    "settings.notification_event_rule_activated": "Rule activated",
+    "settings.notification_event_rule_restored": "Rule restored",
+    "settings.notification_event_rule_pending_deletion": "Rule pending deletion",
+    "settings.notification_event_rule_deleted": "Rule deleted",
+    "settings.notification_event_tunnel_failure": "Cloudflare tunnel update failure",
+    "settings.notification_event_dns_failure": "Cloudflare DNS failure",
+    "settings.notification_event_access_failure": "Cloudflare Access failure",
+    "settings.notification_event_docker_listener_failure": "Docker listener stopped",
+    "settings.notification_event_agent_offline": "Agent offline",
+    "settings.notification_event_agent_online": "Agent recovered",
+    "settings.notification_event_tunnel_down": "Tunnel down",
+    "settings.notification_event_tunnel_recovered": "Tunnel recovered",
+    "settings.save_notification_settings": "Save notification settings",
+    "settings.send_test_notification": "Send test notification",
+    "settings.notification_security_warning": "Apprise URLs often contain secrets and can make DockFlare connect to external or internal addresses. Configure only destinations you trust. Backups contain the encrypted configuration and its key and must be stored securely.",
+    "settings.test_notification_pending": "Sending test notification...",
+    "settings.test_notification_success": "Test notification delivered successfully.",
+    "settings.test_notification_failed": "Test notification failed. Check DockFlare logs.",
+    "settings.test_notification_timeout": "Test notification is still pending. Check DockFlare logs.",
     "settings.tunnel_agent_status": "Tunnel & Agent Status",
     "settings.check_version": "Check version",
     "settings.check_version_title": "Check DockFlare version",
@@ -774,6 +806,11 @@
     "flash.setup.settings_confirmed": "Settings confirmed. Please create an admin user to continue.",
     "flash.setup.required_fields_missing": "Warning: Missing required fields (CF_API_TOKEN or CF_ACCOUNT_ID). You will not be able to proceed.",
     "flash.setup.setup_complete": "Setup complete! Please log in to continue.",
+    "flash.notification_settings_invalid": "Notification settings are invalid.",
+    "flash.notification_settings_save_failed": "Failed to save notification settings.",
+    "flash.notification_destination_invalid": "Invalid Apprise destination: {detail}.",
+    "flash.notification_destination_required": "At least one Apprise destination is required before notifications can be enabled.",
+    "flash.notification_settings_saved": "Notification settings saved.",
     "form.setup.username": "Username",
     "form.setup.password": "Password",
     "form.setup.confirm_password": "Confirm Password",

dockflare/app/main.py

@@ -23,6 +23,7 @@ import sys
 import os
 import json
 import secrets
+import copy
 from cryptography.fernet import Fernet
 
 from app import app, docker_client, tunnel_state, cloudflared_agent_state, config
@@ -36,11 +37,13 @@ from app.core.tunnel_manager import (
 from app.core.docker_handler import start_event_listeners, process_container_start
 from app.core.reconciler import cleanup_expired_rules, reconcile_state_threaded
 from app.core.agent_decommission import timeout_worker as agent_decommission_timeout_worker
+from app.core import notification_manager
 
 stop_event = threading.Event()
 background_threads_list = []
 agent_status_updater_thread = None
 main_initialization_thread = None
+notification_health_thread = None
 
 
 def _restart_if_requested():
@@ -55,7 +58,7 @@ def _restart_if_requested():
         sys.exit(0)
 
 def run_all_background_tasks():
-    global background_threads_list, agent_status_updater_thread 
+    global background_threads_list, agent_status_updater_thread, notification_health_thread
     
     threads_to_start = []
     if not docker_client:
@@ -86,6 +89,14 @@ def run_all_background_tasks():
         logging.info("Starting periodic agent status updater thread...")
         agent_status_updater_thread = threading.Thread(target=periodic_agent_status_updater, name="AgentStatusUpdater", daemon=True)
         threads_to_start.append(agent_status_updater_thread)
+
+    if notification_health_thread is None or not notification_health_thread.is_alive():
+        notification_health_thread = threading.Thread(
+            target=periodic_notification_health_monitor,
+            name="NotificationHealthMonitor",
+            daemon=True,
+        )
+        threads_to_start.append(notification_health_thread)
     
     for t in threads_to_start:
         t.start()
@@ -108,12 +119,62 @@ def periodic_agent_status_updater():
         stop_event.wait(config.AGENT_STATUS_UPDATE_INTERVAL_SECONDS)
     logging.info("Periodic agent status updater task stopped.")
 
+
+def periodic_notification_health_monitor():
+    from app.core.state_manager import agents, agent_decommissions, state_lock
+
+    logging.info("Notification health monitor starting...")
+    while not stop_event.is_set():
+        try:
+            with state_lock:
+                agents_snapshot = copy.deepcopy(agents)
+                decommissioning_ids = {
+                    operation.get("agent_id")
+                    for operation in agent_decommissions.values()
+                    if operation.get("state") not in {"completed", "failed", "cancelled"}
+                }
+                local_status = cloudflared_agent_state.get("container_status")
+                local_tunnel_id = tunnel_state.get("id") or config.EXTERNAL_TUNNEL_ID
+                local_tunnel_name = tunnel_state.get("name")
+            notification_manager.check_agent_health(
+                agents_snapshot,
+                config.AGENT_HEARTBEAT_TIMEOUT,
+                decommissioning_ids=decommissioning_ids,
+            )
+            if not config.USE_EXTERNAL_CLOUDFLARED:
+                notification_manager.check_tunnel_health(
+                    local_tunnel_id,
+                    local_status,
+                    {"tunnel_id": str(local_tunnel_id or "")[:12], "tunnel_name": local_tunnel_name or ""},
+                )
+            for agent_id, agent in agents_snapshot.items():
+                assigned_tunnel_id = agent.get("assigned_tunnel_id")
+                tunnel_status = (agent.get("tunnel_status") or {}).get("status")
+                if assigned_tunnel_id and tunnel_status:
+                    notification_manager.check_tunnel_health(
+                        assigned_tunnel_id,
+                        tunnel_status,
+                        {
+                            "tunnel_id": str(assigned_tunnel_id)[:12],
+                            "tunnel_name": agent.get("assigned_tunnel_name") or "",
+                            "agent_id": str(agent_id)[:12],
+                            "agent_name": agent.get("display_name") or "",
+                        },
+                        intentional=agent_id in decommissioning_ids,
+                    )
+        except Exception:
+            logging.exception("Notification health monitor pass failed.")
+        stop_event.wait(config.NOTIFICATION_HEALTH_INTERVAL_SECONDS)
+    logging.info("Notification health monitor stopped.")
+
 def start_core_services():
     global background_threads_list 
 
     logging.info("Core services initialization process started.")
     if not docker_client:
         logging.error("Docker client unavailable. Critical functionalities will be affected.")
+        notification_manager.end_bootstrap()
+        run_all_background_tasks()
         return
 
     initialize_tunnel()
@@ -246,6 +307,7 @@ def start_core_services():
         except Exception as e:
             logging.debug(f"Could not refresh email sending status on startup: {e}")
 
+    notification_manager.end_bootstrap()
     run_all_background_tasks()
 
 def perform_initial_setup_and_tasks():
@@ -318,6 +380,10 @@ def main_application_entrypoint():
 
     load_state()
     logging.info("Initial state loading from file complete.")
+    notification_manager.configure(config.NOTIFICATION_CONFIG)
+    notification_manager.begin_bootstrap()
+    notification_worker = notification_manager.start(stop_event)
+    background_threads_list.append(notification_worker)
     decommission_timeout_thread = threading.Thread(
         target=agent_decommission_timeout_worker,
         args=(stop_event,),
@@ -425,6 +491,7 @@ def main_application_entrypoint():
     finally:
         logging.info("Shutdown sequence initiated...")
         stop_event.set() 
+        notification_manager.stop(timeout_seconds=5)
         
         if main_initialization_thread and main_initialization_thread.is_alive():
             logging.info("Waiting for main initialization thread to complete (timeout 15s)...")

dockflare/app/templates/docs/en/Apprise-Notifications.md

@@ -0,0 +1,41 @@
+# Apprise Notifications
+
+DockFlare can send operational notifications to services supported by [Apprise](https://github.com/caronc/apprise), including chat, push, email, and webhook providers. Notifications are disabled by default and delivery is best-effort; a destination failure never changes the result of the DockFlare operation that produced the event.
+
+## Configure notifications
+
+1. Open **Settings** and select **Notifications**.
+2. Paste one complete Apprise URL per line into **Replacement Apprise URLs**.
+3. Select the events you want to receive.
+4. Enable notifications and save.
+5. Select **Send test notification** and wait for the delivery result.
+
+The replacement field is intentionally blank whenever the page loads. Leaving it blank retains the currently saved destinations. Entering URLs replaces the entire list. To remove the saved destinations, select **Clear configured destinations** and save.
+
+Consult the [Apprise service documentation](https://github.com/caronc/apprise/wiki) for provider-specific URL formats. Keep these URLs secret: they commonly contain webhook tokens, API keys, usernames, or passwords.
+
+## Available events
+
+DockFlare can report:
+
+- rule activation, restoration, pending deletion, and completed deletion;
+- Cloudflare tunnel configuration, DNS, and Access failures;
+- an unexpected Docker listener failure;
+- Agent offline and recovery transitions; and
+- managed tunnel down and recovery transitions.
+
+Repeated failure notifications for the same resource are limited by the configured cooldown. Recovery notifications are sent only when DockFlare previously emitted the corresponding outage notification. Normal startup scans and intentional Agent or tunnel stops are suppressed to avoid misleading alerts.
+
+## Delivery behavior
+
+Notifications are queued in memory and sent by a background worker, so a slow destination does not block Docker or Cloudflare processing. The queue is bounded; when it is full, new notification events are dropped and counted in the notification status. Queued messages can be lost if DockFlare exits before delivery, and failed deliveries are not automatically retried because some destinations may already have received the message.
+
+The settings page displays only redacted destination summaries. Complete URLs remain in DockFlare's encrypted configuration and are included in encrypted backups, so backups must also be handled as secrets.
+
+## Troubleshooting
+
+- If testing is unavailable, save at least one valid destination and enable notifications first.
+- If a test fails, review DockFlare logs for the event ID and aggregate delivery result. Destination URLs are not written to logs.
+- Confirm the DockFlare container can resolve DNS and reach the destination over the required outbound port.
+- Verify the provider credentials and URL syntax against the Apprise documentation.
+- A restart clears queued messages, cooldown state, health baselines, and recent test-job results.

dockflare/app/templates/docs/en/navigation.md

@@ -20,6 +20,7 @@ This documentation provides comprehensive information for DockFlare. Whether you
     *   [State-Persistence](State-Persistence.md)
 *   **Configuration**
     *   [Container Labels](Container-Labels.md)
+    *   [Apprise Notifications](Apprise-Notifications.md)
     *   [Identity Providers](Identity-Providers.md)
     *   [OAuth Provider Setup](OAuth-Provider-Setup.md)
 *   **Usage Guide**

dockflare/app/templates/settings.html

@@ -18,6 +18,7 @@
                 <li><a href="{{ url_for('web.settings_page') }}#backup-restore">{{ t('settings.backup_restore') }}</a></li>
                 <li><a href="{{ url_for('web.settings_page') }}#security">{{ t('settings.security') }}</a></li>
                 <li><a href="{{ url_for('web.settings_page') }}#security" class="pl-8 text-sm">{{ t('settings.oauth_authentication') }}</a></li>
+                <li><a href="{{ url_for('web.settings_page') }}#notifications">{{ t('settings.notifications') }}</a></li>
                 <li><a href="{{ url_for('web.settings_page') }}#tunnel-agent-status">{{ t('settings.tunnel_agent_status') }}</a></li>
             </ul>
             <div class="mt-4 px-2">
@@ -452,7 +453,100 @@
             </div>
         </section>
 
-        <!-- 5. Tunnel & Agent Status Section -->
+        <section id="notifications" class="card bg-base-100 shadow-xl mb-8 sm:mb-12 transition-all duration-300 hover:shadow-2xl">
+            <div class="card-body">
+                <h2 class="card-title text-2xl sm:text-3xl border-b border-base-300 pb-3 mb-6">
+                    {{ t('settings.notifications') }}
+                </h2>
+                <p class="text-sm opacity-70">{{ t('settings.notifications_description') }}</p>
+
+                <div class="rounded-box bg-base-200 p-4 mt-4 text-sm space-y-2">
+                    <p><strong>{{ t('settings.notification_status') }}:</strong>
+                        {% if notification_status.enabled and notification_status.available %}
+                            <span class="badge badge-success">{{ t('common.active') }}</span>
+                        {% elif notification_status.enabled %}
+                            <span class="badge badge-error">{{ t('settings.notification_unavailable') }}</span>
+                        {% else %}
+                            <span class="badge badge-ghost">{{ t('settings.notification_disabled') }}</span>
+                        {% endif %}
+                    </p>
+                    <p><strong>{{ t('settings.configured_destinations') }}:</strong> {{ notification_status.configured_destination_count }}</p>
+                    {% if notification_status.destinations %}
+                        <ul class="list-disc list-inside opacity-80">
+                            {% for destination in notification_status.destinations %}
+                                <li>{{ destination.summary }}</li>
+                            {% endfor %}
+                        </ul>
+                    {% endif %}
+                </div>
+
+                <form method="POST" action="{{ url_for('web.save_notification_settings') }}" class="space-y-5 mt-5">
+                    {{ notification_settings_form.hidden_tag() }}
+                    <div class="form-control">
+                        <label class="label cursor-pointer justify-start gap-3">
+                            {{ notification_settings_form.enabled(class="toggle toggle-primary") }}
+                            <span class="label-text">{{ t('settings.enable_notifications') }}</span>
+                        </label>
+                    </div>
+                    <div class="form-control">
+                        <label class="label" for="notification-replacement-urls">
+                            <span class="label-text">{{ t('settings.replacement_apprise_urls') }}</span>
+                        </label>
+                        {{ notification_settings_form.replacement_urls(class="textarea textarea-bordered min-h-32 font-mono text-sm", id="notification-replacement-urls", placeholder="discord://...\ntgram://...") }}
+                        <label class="label"><span class="label-text-alt">{{ t('settings.replacement_apprise_urls_help') }}</span></label>
+                    </div>
+                    <div class="form-control">
+                        <label class="label cursor-pointer justify-start gap-3">
+                            {{ notification_settings_form.clear_urls(class="checkbox checkbox-error") }}
+                            <span class="label-text">{{ t('settings.clear_notification_destinations') }}</span>
+                        </label>
+                    </div>
+                    <div class="form-control max-w-xs">
+                        <label class="label" for="notification-cooldown"><span class="label-text">{{ t('settings.notification_cooldown') }}</span></label>
+                        {{ notification_settings_form.failure_cooldown_seconds(class="input input-bordered", id="notification-cooldown", min="60", max="86400") }}
+                        <label class="label"><span class="label-text-alt">{{ t('settings.notification_cooldown_help') }}</span></label>
+                    </div>
+
+                    {% set notification_events = [
+                        ('rule_activated', 'settings.notification_event_rule_activated'),
+                        ('rule_restored', 'settings.notification_event_rule_restored'),
+                        ('rule_pending_deletion', 'settings.notification_event_rule_pending_deletion'),
+                        ('rule_deleted', 'settings.notification_event_rule_deleted'),
+                        ('cloudflare_tunnel_failure', 'settings.notification_event_tunnel_failure'),
+                        ('cloudflare_dns_failure', 'settings.notification_event_dns_failure'),
+                        ('cloudflare_access_failure', 'settings.notification_event_access_failure'),
+                        ('docker_listener_failure', 'settings.notification_event_docker_listener_failure'),
+                        ('agent_offline', 'settings.notification_event_agent_offline'),
+                        ('agent_online', 'settings.notification_event_agent_online'),
+                        ('tunnel_down', 'settings.notification_event_tunnel_down'),
+                        ('tunnel_recovered', 'settings.notification_event_tunnel_recovered')
+                    ] %}
+                    <fieldset>
+                        <legend class="font-semibold mb-3">{{ t('settings.notification_events') }}</legend>
+                        <div class="grid grid-cols-1 md:grid-cols-2 gap-3">
+                            {% for field_name, label_key in notification_events %}
+                                <label class="label cursor-pointer justify-start gap-3 rounded-md bg-base-200/50 px-3">
+                                    {{ notification_settings_form[field_name](class="checkbox checkbox-sm checkbox-primary") }}
+                                    <span class="label-text">{{ t(label_key) }}</span>
+                                </label>
+                            {% endfor %}
+                        </div>
+                    </fieldset>
+
+                    <div class="card-actions pt-2 gap-3">
+                        {{ notification_settings_form.submit_notifications(class="btn btn-primary", value=t('settings.save_notification_settings')) }}
+                        <button type="button" id="btn-test-notifications" class="btn btn-outline" {% if not notification_status.enabled or not notification_status.available %}disabled{% endif %}>{{ t('settings.send_test_notification') }}</button>
+                    </div>
+                    <div id="notification-test-status" class="text-sm hidden" role="status" aria-live="polite"></div>
+                </form>
+
+                <div role="alert" class="alert alert-warning text-sm mt-5">
+                    <span>{{ t('settings.notification_security_warning') }}</span>
+                </div>
+            </div>
+        </section>
+
+        <!-- 6. Tunnel & Agent Status Section -->
         <section id="tunnel-agent-status" class="card bg-base-100 shadow-xl mb-8 sm:mb-12 transition-all duration-300 hover:shadow-2xl">
             <div class="card-body">
                 <h2 class="card-title text-2xl sm:text-3xl border-b border-base-300 pb-3 mb-6">
@@ -536,6 +630,55 @@
 <script>
   (function () {
 
+    const testNotificationsBtn = document.getElementById('btn-test-notifications');
+    const notificationTestStatus = document.getElementById('notification-test-status');
+    if (testNotificationsBtn && notificationTestStatus) {
+      const setNotificationTestStatus = (message, isError) => {
+        notificationTestStatus.textContent = message;
+        notificationTestStatus.classList.remove('hidden', 'text-error', 'text-success');
+        notificationTestStatus.classList.add(isError ? 'text-error' : 'text-success');
+      };
+      const pollNotificationTest = async (jobId) => {
+        for (let attempt = 0; attempt < 30; attempt += 1) {
+          await new Promise(resolve => setTimeout(resolve, 1000));
+          const response = await fetch(`/api/v2/notifications/test/${encodeURIComponent(jobId)}`, {credentials: 'same-origin'});
+          const payload = await response.json().catch(() => ({}));
+          if (payload.status === 'success') {
+            setNotificationTestStatus("{{ t('settings.test_notification_success') }}", false);
+            return;
+          }
+          if (!response.ok || payload.status === 'failure' || payload.status === 'not_found') {
+            setNotificationTestStatus("{{ t('settings.test_notification_failed') }}", true);
+            return;
+          }
+        }
+        setNotificationTestStatus("{{ t('settings.test_notification_timeout') }}", true);
+      };
+      testNotificationsBtn.addEventListener('click', async () => {
+        testNotificationsBtn.disabled = true;
+        notificationTestStatus.textContent = "{{ t('settings.test_notification_pending') }}";
+        notificationTestStatus.classList.remove('hidden', 'text-error', 'text-success');
+        try {
+          const response = await fetch('/api/v2/notifications/test', {
+            method: 'POST',
+            headers: {'Content-Type': 'application/json', 'X-CSRFToken': '{{ csrf_token() }}'},
+            credentials: 'same-origin',
+            body: '{}'
+          });
+          const payload = await response.json().catch(() => ({}));
+          if (!response.ok || !payload.job_id) {
+            setNotificationTestStatus("{{ t('settings.test_notification_failed') }}", true);
+            return;
+          }
+          await pollNotificationTest(payload.job_id);
+        } catch (error) {
+          setNotificationTestStatus("{{ t('settings.test_notification_failed') }}", true);
+        } finally {
+          testNotificationsBtn.disabled = false;
+        }
+      });
+    }
+
     const showMasterKeyBtn = document.getElementById('btn-show-master-key');
     if (showMasterKeyBtn) {
       showMasterKeyBtn.addEventListener('click', async () => {

dockflare/app/web/api_v2_routes.py

@@ -37,6 +37,7 @@ from app.core.state_manager import (
 )
 from app.core import agent_key_store
 from app.core import agent_decommission
+from app.core import notification_manager
 from app.core.tunnel_manager import (
     start_cloudflared_container,
     stop_cloudflared_container,
@@ -536,6 +537,7 @@ def create_manual_rule_api():
             save_state()
             state_changed = True
 
+    access_success = True
     if access_group_ids_list:
         from app import config
         rule = managed_rules.get(rule_key)
@@ -585,6 +587,7 @@ def create_manual_rule_api():
                             logging.info(f"API: Successfully updated Access Application for {rule_key}")
                         else:
                             logging.error(f"API: Failed to update Access Application for {rule_key}")
+                            access_success = False
                     else:
                         logging.info(f"API: Creating new Access Application for rule {rule_key}")
                         from app.core.access_manager import create_cloudflare_access_application
@@ -607,8 +610,17 @@ def create_manual_rule_api():
                             logging.info(f"API: Created Access Application ID '{new_app_id}' for {rule_key}")
                         else:
                             logging.error(f"API: Failed to create Access Application for {rule_key}")
+                            access_success = False
                 except Exception as e:
                     logging.error(f"API: Error creating/updating Access Application for {rule_key}: {e}", exc_info=True)
+                    access_success = False
+
+        if not access_success:
+            notification_manager.emit(
+                "cloudflare.access_failure",
+                rule_key,
+                {"hostname": hostname, "path": normalized_path, "operation": "apply policy", "source": "manual"},
+            )
 
     if state_changed:
         publish_state_event('snapshot_refresh')
@@ -628,6 +640,18 @@ def create_manual_rule_api():
         update_cloudflare_config(master_tunnel_id)
     dns_success = bool(dns_result) and dns_result not in {"semaphore_timeout", "existing_record_unconfirmed"}
     if not dns_success or not tunnel_update_success:
+        if not dns_success:
+            notification_manager.emit(
+                "cloudflare.dns_failure",
+                f"{zone_id}:{hostname}",
+                {"hostname": hostname, "path": normalized_path, "operation": "create", "source": "manual"},
+            )
+        if not tunnel_update_success:
+            notification_manager.emit(
+                "cloudflare.tunnel_failure",
+                tunnel_id,
+                {"hostname": hostname, "tunnel_id": tunnel_id[:12], "tunnel_name": tunnel_name, "operation": "update", "source": "manual"},
+            )
         with state_lock:
             if previous_rule_snapshot is None:
                 managed_rules.pop(rule_key, None)
@@ -673,6 +697,18 @@ def create_manual_rule_api():
         if old_tuple != new_tuple and not old_tuple_still_owned and all(old_tuple) and not old_tuple[0].startswith('*.'):
             delete_cloudflare_dns_record(old_tuple[1], old_tuple[0], old_tuple[2])
     status_code = 201 if state_changed else 200
+    if previous_rule_snapshot is None and state_changed and access_success:
+        notification_manager.emit(
+            "rule.activated",
+            rule_key,
+            {
+                "source": "manual",
+                "tunnel_id": tunnel_id[:12],
+                "tunnel_name": tunnel_name,
+                "resources": [{"key": rule_key, "hostname": hostname, "path": normalized_path, "source": "manual"}],
+                "public_url": config.DOCKFLARE_PUBLIC_URL,
+            },
+        )
     return jsonify({"rule_key": rule_key}), status_code
 
 @api_v2_bp.route('/reconciliation-status', methods=['GET'])
@@ -923,6 +959,8 @@ def process_agent_container_start(payload, agent_id, defer_side_effects=False):
 
             state_changed_locally = False
             needs_tunnel_config_update = False
+            activated_resources = []
+            restored_resources = []
 
             agent_record = get_agent(agent_id)
             assigned_tunnel_name = agent_record.get("assigned_tunnel_name") if agent_record else "Unknown"
@@ -951,6 +989,12 @@ def process_agent_container_start(payload, agent_id, defer_side_effects=False):
                         if reactivated:
                             mark_rule_tunnel_sync_pending(matched_rule)
                             needs_tunnel_config_update = True
+                            restored_resources.append({
+                                "key": matched_key,
+                                "hostname": matched_rule.get("hostname"),
+                                "path": matched_rule.get("path"),
+                                "source": "agent",
+                            })
                         state_changed_locally |= changed
                         if matched_rule.get("hostname") and matched_rule.get("zone_id"):
                             dns_targets[matched_rule["hostname"]] = {"zone_id": matched_rule["zone_id"]}
@@ -981,6 +1025,11 @@ def process_agent_container_start(payload, agent_id, defer_side_effects=False):
                     )
                 except ZoneResolutionError as exc:
                     logging.error(f"AGENT_PROCESS: Zone resolution failed for {rule_key} ({exc.code}). Skipping.")
+                    notification_manager.emit(
+                        "cloudflare.dns_failure",
+                        rule_key,
+                        {"hostname": hostname, "path": path_from_item, "operation": "zone resolution", "source": "agent", "agent_id": agent_id[:12]},
+                    )
                     continue
                 target_zone_id = selected_zone["id"]
                 zone_name_from_item = selected_zone.get("name")
@@ -1056,6 +1105,12 @@ def process_agent_container_start(payload, agent_id, defer_side_effects=False):
                             existing_rule["status"] = "active"
                             existing_rule["delete_at"] = None
                             rule_data_changed = True
+                            restored_resources.append({
+                                "key": rule_key,
+                                "hostname": hostname,
+                                "path": path_from_item,
+                                "source": "agent",
+                            })
 
                         tunnel_fields = (
                             "hostname", "path", "service", "zone_id", "no_tls_verify",
@@ -1109,6 +1164,12 @@ def process_agent_container_start(payload, agent_id, defer_side_effects=False):
                         existing_rule = managed_rules[rule_key]
                         state_changed_locally = True
                         needs_tunnel_config_update = True
+                        activated_resources.append({
+                            "key": rule_key,
+                            "hostname": hostname,
+                            "path": path_from_item,
+                            "source": "agent",
+                        })
 
                     dns_targets[hostname] = {
                         "zone_id": target_zone_id,
@@ -1129,6 +1190,11 @@ def process_agent_container_start(payload, agent_id, defer_side_effects=False):
                 "policy_jobs": policy_jobs,
                 "dns_targets": dns_targets,
                 "tunnel_id": assigned_tunnel_id,
+                "tunnel_name": assigned_tunnel_name,
+                "container_id": container_id,
+                "container_name": container_name,
+                "activated_resources": activated_resources,
+                "restored_resources": restored_resources,
             }
             if defer_side_effects:
                 return plan
@@ -1149,8 +1215,16 @@ def _execute_agent_start_plan(plan):
         return
     policy_changed = False
     for rule_key, policy_payload in plan.get("policy_jobs", []):
-        if handle_access_policy_from_labels(rule_key, copy.deepcopy(policy_payload)):
-            policy_changed = True
+        try:
+            if handle_access_policy_from_labels(rule_key, copy.deepcopy(policy_payload)):
+                policy_changed = True
+        except Exception:
+            notification_manager.emit(
+                "cloudflare.access_failure",
+                rule_key,
+                {"hostname": policy_payload.get("hostname"), "path": policy_payload.get("path"), "operation": "apply policy", "source": "agent", "agent_id": str(plan.get("agent_id") or "")[:12]},
+            )
+            raise
     if policy_changed:
         save_state()
         publish_state_event('snapshot_refresh')
@@ -1161,17 +1235,45 @@ def _execute_agent_start_plan(plan):
     agent_tunnel_id = plan.get("tunnel_id")
     if not agent_tunnel_id:
         logging.error("AGENT_PROCESS: Agent %s has no tunnel ID; synchronization remains pending.", agent_id)
+        notification_manager.emit(
+            "cloudflare.tunnel_failure",
+            agent_id,
+            {"agent_id": str(agent_id)[:12], "operation": "resolve tunnel", "source": "agent"},
+        )
         return
     try:
         if not update_cloudflare_config(agent_tunnel_id):
             logging.error("AGENT_PROCESS: Failed to update tunnel config for agent %s", agent_id)
+            notification_manager.emit(
+                "cloudflare.tunnel_failure",
+                agent_tunnel_id,
+                {"agent_id": str(agent_id)[:12], "tunnel_id": str(agent_tunnel_id)[:12], "tunnel_name": plan.get("tunnel_name"), "operation": "update", "source": "agent"},
+            )
             return
+        remote_success = True
         for hostname, details in plan.get("dns_targets", {}).items():
             zone_id = details.get("zone_id")
             if zone_id and not hostname.startswith('*.'):
-                create_cloudflare_dns_record(zone_id, hostname, agent_tunnel_id)
+                try:
+                    dns_result = create_cloudflare_dns_record(zone_id, hostname, agent_tunnel_id)
+                except Exception:
+                    logging.exception("AGENT_PROCESS: DNS creation raised for %s", hostname)
+                    dns_result = None
+                if not dns_result or dns_result in {"semaphore_timeout", "existing_record_unconfirmed"}:
+                    remote_success = False
+                    notification_manager.emit(
+                        "cloudflare.dns_failure",
+                        f"{zone_id}:{hostname}",
+                        {"hostname": hostname, "operation": "create", "source": "agent", "agent_id": str(agent_id)[:12]},
+                    )
             elif not zone_id:
                 logging.error("AGENT_PROCESS: Could not determine Zone ID for DNS record %s", hostname)
+                remote_success = False
+                notification_manager.emit(
+                    "cloudflare.dns_failure",
+                    hostname,
+                    {"hostname": hostname, "operation": "create", "source": "agent", "agent_id": str(agent_id)[:12]},
+                )
         sync_changed = False
         with state_lock:
             for rule in managed_rules.values():
@@ -1184,8 +1286,27 @@ def _execute_agent_start_plan(plan):
         if sync_changed:
             save_state()
         logging.info("AGENT_PROCESS: Successfully updated tunnel config for agent %s", agent_id)
+        if remote_success:
+            common_context = {
+                "agent_id": str(agent_id)[:12],
+                "tunnel_id": str(agent_tunnel_id)[:12],
+                "tunnel_name": plan.get("tunnel_name"),
+                "container_id": str(plan.get("container_id") or "")[:12],
+                "container_name": plan.get("container_name"),
+                "source": "agent",
+                "public_url": config.DOCKFLARE_PUBLIC_URL,
+            }
+            if plan.get("activated_resources"):
+                notification_manager.emit("rule.activated", str(plan.get("container_id") or agent_id), {**common_context, "resources": plan["activated_resources"]})
+            if plan.get("restored_resources"):
+                notification_manager.emit("rule.restored", str(plan.get("container_id") or agent_id), {**common_context, "resources": plan["restored_resources"]})
     except Exception:
         logging.exception("AGENT_PROCESS: Failed to update tunnel config for agent %s", agent_id)
+        notification_manager.emit(
+            "cloudflare.tunnel_failure",
+            str(agent_tunnel_id),
+            {"agent_id": str(agent_id)[:12], "tunnel_id": str(agent_tunnel_id)[:12], "operation": "update", "source": "agent"},
+        )
 
 
 def _coalesce_agent_start_plans(plans):
@@ -1200,11 +1321,18 @@ def _coalesce_agent_start_plans(plans):
             "needs_tunnel_config_update": False,
             "policy_jobs": [],
             "dns_targets": {},
+            "tunnel_name": plan.get("tunnel_name"),
+            "container_id": plan.get("container_id"),
+            "container_name": plan.get("container_name"),
+            "activated_resources": [],
+            "restored_resources": [],
         })
         merged["state_changed"] |= bool(plan.get("state_changed"))
         merged["needs_tunnel_config_update"] |= bool(plan.get("needs_tunnel_config_update"))
         merged["policy_jobs"].extend(plan.get("policy_jobs", []))
         merged["dns_targets"].update(plan.get("dns_targets", {}))
+        merged["activated_resources"].extend(plan.get("activated_resources", []))
+        merged["restored_resources"].extend(plan.get("restored_resources", []))
     return list(combined.values())
 
 
@@ -1218,6 +1346,9 @@ def process_agent_container_stop(payload, agent_id, received_at=None, persist=Tr
 
         logging.info(f"AGENT_PROCESS_STOP: Processing stop for container {container_id[:12]} from agent {agent_id}")
 
+        pending_resources = []
+        delete_at_value = None
+        grace_period = current_app.config.get('GRACE_PERIOD_SECONDS', 28800)
         with state_lock:
             rule_keys_affected = []
             for r_key, details in managed_rules.items():
@@ -1228,7 +1359,6 @@ def process_agent_container_stop(payload, agent_id, received_at=None, persist=Tr
                     rule_keys_affected.append(r_key)
 
             if rule_keys_affected:
-                grace_period = current_app.config.get('GRACE_PERIOD_SECONDS', 28800)
                 for rule_key in rule_keys_affected:
                     rule = managed_rules[rule_key]
                     if rule.get("status") != "pending_deletion":
@@ -1237,14 +1367,21 @@ def process_agent_container_stop(payload, agent_id, received_at=None, persist=Tr
                         rule["delete_at"] = (received_at or datetime.now(timezone.utc)) + grace_delta
                         rule["lifecycle_generation"] = int(rule.get("lifecycle_generation") or 0) + 1
                         logging.info(f"AGENT_PROCESS_STOP: Rule for {rule_key} scheduled for deletion (grace period: {grace_period}s)")
+                        delete_at_value = rule["delete_at"].isoformat()
+                        pending_resources.append({"key": rule_key, "hostname": rule.get("hostname"), "path": rule.get("path"), "source": "agent"})
                 if persist:
                     save_state()
                     publish_state_event('snapshot_refresh')
                 logging.info(f"AGENT_PROCESS_STOP: Scheduled {len(rule_keys_affected)} rules for deletion from agent {agent_id}")
-                return True
             else:
                 logging.info(f"AGENT_PROCESS_STOP: No active agent-managed rules found for container {container_id[:12]} from agent {agent_id}")
-        return False
+        if persist and pending_resources:
+            notification_manager.emit(
+                "rule.pending_deletion",
+                container_id,
+                {"container_id": container_id[:12], "agent_id": str(agent_id)[:12], "source": "agent", "delete_at": delete_at_value, "grace_period_seconds": grace_period, "resources": pending_resources, "public_url": config.DOCKFLARE_PUBLIC_URL},
+            )
+        return bool(pending_resources)
 
 @api_v2_bp.route('/agents/generate-key', methods=['POST', 'GET'])
 def agents_generate_key():
@@ -1752,6 +1889,8 @@ def _handle_agent_event_request(agent_id):
         rules_before = copy.deepcopy(managed_rules)
         agents_before = copy.deepcopy(agents)
     side_effect_plans = []
+    pending_notification_resources = []
+    pending_notification_deadline = None
     try:
         if event_type == "container_start":
             plan = process_agent_container_start({"container": valid_containers[0]}, agent_id, defer_side_effects=True)
@@ -1816,7 +1955,39 @@ def _handle_agent_event_request(agent_id):
             agents.clear()
             agents.update(agents_before)
             return _agent_json_response(503, "persistence_failed")
+        for rule_key, rule in managed_rules.items():
+            previous_rule = rules_before.get(rule_key)
+            if (
+                rule.get("source") == "agent"
+                and rule.get("agent_id") == agent_id
+                and rule.get("status") == "pending_deletion"
+                and (not previous_rule or previous_rule.get("status") != "pending_deletion")
+            ):
+                pending_notification_resources.append({
+                    "key": rule_key,
+                    "hostname": rule.get("hostname"),
+                    "path": rule.get("path"),
+                    "source": "agent",
+                })
+                deadline = rule.get("delete_at")
+                if isinstance(deadline, datetime):
+                    pending_notification_deadline = deadline.isoformat()
     publish_state_event("snapshot_refresh")
+    if pending_notification_resources:
+        container_id = valid_containers[0]["id"] if len(valid_containers) == 1 else agent_id
+        notification_manager.emit(
+            "rule.pending_deletion",
+            container_id,
+            {
+                "container_id": str(container_id)[:12],
+                "agent_id": str(agent_id)[:12],
+                "source": "agent",
+                "delete_at": pending_notification_deadline,
+                "grace_period_seconds": current_app.config.get("GRACE_PERIOD_SECONDS", 28800),
+                "resources": pending_notification_resources,
+                "public_url": config.DOCKFLARE_PUBLIC_URL,
+            },
+        )
     for plan in _coalesce_agent_start_plans(side_effect_plans):
         _execute_agent_start_plan(plan)
     http_status = 200 if result_code == "accepted" else 202
@@ -2347,6 +2518,7 @@ def agent_start():
     if not docker_client:
         return jsonify({"status": "error", "message": "Docker client not available."}),
         
+    notification_manager.suppress_tunnel_health(tunnel_state.get("id"), 120)
     start_cloudflared_container()
     time.sleep(0.5)
     return jsonify({
@@ -2362,6 +2534,7 @@ def agent_stop():
     if not docker_client:
         return jsonify({"status": "error", "message": "Docker client not available."}),
 
+    notification_manager.suppress_tunnel_health(tunnel_state.get("id"), 300)
     stop_cloudflared_container()
     time.sleep(0.5)
     return jsonify({
@@ -2430,6 +2603,31 @@ def delete_manual_rule(rule_key):
 
     publish_state_event('snapshot_refresh')
 
+    if not dns_deleted_ok:
+        notification_manager.emit(
+            "cloudflare.dns_failure",
+            f"{zone_id_for_delete}:{hostname_for_dns_operations}",
+            {"hostname": hostname_for_dns_operations, "operation": "delete", "source": "manual"},
+        )
+    if not access_app_deleted_ok:
+        notification_manager.emit(
+            "cloudflare.access_failure",
+            rule_key,
+            {"hostname": hostname_for_dns_operations, "operation": "delete", "source": "manual"},
+        )
+    if not config_update_success:
+        notification_manager.emit(
+            "cloudflare.tunnel_failure",
+            str(tunnel_id_for_delete),
+            {"hostname": hostname_for_dns_operations, "tunnel_id": str(tunnel_id_for_delete)[:12], "operation": "delete rule update", "source": "manual"},
+        )
+    if config_update_success and dns_deleted_ok and access_app_deleted_ok:
+        notification_manager.emit(
+            "rule.deleted",
+            rule_key,
+            {"source": "manual", "resources": [{"key": rule_key, "hostname": hostname_for_dns_operations, "source": "manual"}], "public_url": config.DOCKFLARE_PUBLIC_URL},
+        )
+
     if config_update_success:
         message = f"Manual rule {rule_key} deleted."
         if not dns_deleted_ok:
@@ -2503,6 +2701,24 @@ def force_delete_rule(rule_key):
     if not all(results.values()):
         status_code = 207 
 
+    notification_context = {
+        "hostname": hostname_for_dns,
+        "operation": "force delete",
+        "source": rule_details_copy.get("source") or "manual",
+    }
+    if not dns_deleted:
+        notification_manager.emit("cloudflare.dns_failure", f"{zone_id_for_delete}:{hostname_for_dns}", notification_context)
+    if not access_app_deleted:
+        notification_manager.emit("cloudflare.access_failure", rule_key, notification_context)
+    if not config_updated:
+        notification_manager.emit("cloudflare.tunnel_failure", effective_tunnel_id, notification_context)
+    if status_code == 200:
+        notification_manager.emit(
+            "rule.deleted",
+            rule_key,
+            {**notification_context, "resources": [{"key": rule_key, "hostname": hostname_for_dns, "source": notification_context["source"]}], "public_url": config.DOCKFLARE_PUBLIC_URL},
+        )
+
     return jsonify({
         "status": "success" if status_code == 200 else "warning",
         "message": f"Rule '{rule_key}' force deleted. See details.",

dockflare/app/web/config_loader.py

@@ -20,12 +20,14 @@
 import json
 import logging
 import os
+import tempfile
 from typing import Dict, Optional
 
 from cryptography.fernet import Fernet
 
 from app import config
 from app.core.container_name import build_cloudflared_container_name
+from app.core.notification_manager import normalize_notification_config
 
 
 def _data_directory() -> str:
@@ -67,6 +69,36 @@ def load_encrypted_config() -> Optional[Dict]:
     return data
 
 
+def save_encrypted_config(config_data: Dict, fernet: Fernet) -> bool:
+    if not isinstance(config_data, dict) or fernet is None:
+        return False
+    destination = config_file_path()
+    temporary_path = None
+    try:
+        payload = fernet.encrypt(json.dumps(config_data).encode("utf-8"))
+        os.makedirs(os.path.dirname(destination), exist_ok=True)
+        with tempfile.NamedTemporaryFile(
+            mode="wb",
+            dir=os.path.dirname(destination),
+            prefix=".dockflare_config.",
+            delete=False,
+        ) as temporary:
+            temporary.write(payload)
+            temporary.flush()
+            os.fsync(temporary.fileno())
+            temporary_path = temporary.name
+        os.replace(temporary_path, destination)
+        return True
+    except Exception as err:
+        logging.error("CONFIG_LOADER: Failed to save encrypted config: %s", type(err).__name__)
+        if temporary_path:
+            try:
+                os.unlink(temporary_path)
+            except OSError:
+                pass
+        return False
+
+
 def apply_config_to_app(flask_app, config_data: Dict) -> None:
     """Populate Flask app and module-level config values from decrypted data."""
     if not isinstance(config_data, dict):
@@ -134,6 +166,10 @@ def apply_config_to_app(flask_app, config_data: Dict) -> None:
     config.EMAIL_ENABLED = flask_app.config['EMAIL_ENABLED']
     config.EMAIL_CONFIG = flask_app.config['EMAIL_CONFIG']
 
+    notification_config = normalize_notification_config(config_data.get('notification_config'))
+    flask_app.config['NOTIFICATION_CONFIG'] = notification_config
+    config.NOTIFICATION_CONFIG = notification_config
+
     if flask_app.config['CF_API_TOKEN']:
         config.CF_HEADERS['Authorization'] = f"Bearer {flask_app.config['CF_API_TOKEN']}"
     else:

dockflare/app/web/forms.py

@@ -16,8 +16,8 @@
 #
 # dockflare/app/web/forms.py
 from flask_wtf import FlaskForm
-from wtforms import BooleanField, PasswordField, SubmitField, StringField, IntegerField
-from wtforms.validators import DataRequired, EqualTo, Length, Optional
+from wtforms import BooleanField, PasswordField, SubmitField, StringField, IntegerField, TextAreaField
+from wtforms.validators import DataRequired, EqualTo, Length, NumberRange, Optional
 
 class SettingsForm(FlaskForm):
     """Form for editing general application settings."""
@@ -109,3 +109,27 @@ class CloudflareCredentialsForm(FlaskForm):
         validators=[Optional(), Length(min=40, max=100, message="API Token must be at least 40 characters long.")]
     )
     submit_cloudflare_credentials = SubmitField('Update Cloudflare Credentials')
+
+
+class NotificationSettingsForm(FlaskForm):
+    enabled = BooleanField('Enable notifications')
+    replacement_urls = TextAreaField('Replacement Apprise URLs', validators=[Optional(), Length(max=131072)])
+    clear_urls = BooleanField('Clear configured destinations')
+    failure_cooldown_seconds = IntegerField(
+        'Failure cooldown (seconds)',
+        validators=[DataRequired(), NumberRange(min=60, max=86400)],
+        default=900,
+    )
+    rule_activated = BooleanField('Rule activated')
+    rule_restored = BooleanField('Rule restored')
+    rule_pending_deletion = BooleanField('Rule pending deletion')
+    rule_deleted = BooleanField('Rule deleted')
+    cloudflare_tunnel_failure = BooleanField('Cloudflare tunnel failure')
+    cloudflare_dns_failure = BooleanField('Cloudflare DNS failure')
+    cloudflare_access_failure = BooleanField('Cloudflare Access failure')
+    docker_listener_failure = BooleanField('Docker listener failure')
+    agent_offline = BooleanField('Agent offline')
+    agent_online = BooleanField('Agent recovered')
+    tunnel_down = BooleanField('Tunnel down')
+    tunnel_recovered = BooleanField('Tunnel recovered')
+    submit_notifications = SubmitField('Save notification settings')

dockflare/app/web/routes.py

@@ -87,6 +87,7 @@ from app.core.docker_handler import is_valid_hostname, is_valid_service
 from app.core.utils import get_rule_key, normalize_path_value
 from app.core.container_name import build_cloudflared_container_name
 from app.core import backup_manager
+from app.core import notification_manager
 from app.web import config_loader
 from app.i18n import t as _t
 from cryptography.fernet import Fernet
@@ -465,7 +466,13 @@ def status_page():
                         default_tunnel_id=default_tunnel_id_value
                         )
 
-from app.web.forms import ChangePasswordForm, SecuritySettingsForm, SettingsForm, CloudflareCredentialsForm
+from app.web.forms import (
+    ChangePasswordForm,
+    CloudflareCredentialsForm,
+    NotificationSettingsForm,
+    SecuritySettingsForm,
+    SettingsForm,
+)
 from werkzeug.security import check_password_hash, generate_password_hash
 from cryptography.fernet import Fernet
 
@@ -556,6 +563,7 @@ def settings_page():
     change_password_form = ChangePasswordForm()
     security_settings_form = SecuritySettingsForm(prefix='security')
     cf_credentials_form = CloudflareCredentialsForm(prefix='cf_creds')
+    notification_settings_form = NotificationSettingsForm(prefix='notifications')
 
     
     if request.method == 'POST':
@@ -612,6 +620,7 @@ def settings_page():
                 if tunnel_name_changed and not config.USE_EXTERNAL_CLOUDFLARED:
                     flash(_t('flash.tunnel_name_changed'), 'info')
                     logging.info(f"Tunnel name changed from '{original_tunnel_name}' to '{new_tunnel_name}'. Triggering agent restart.")
+                    notification_manager.suppress_tunnel_health(tunnel_state.get("id"), 300)
                     
                     def restart_agent_task():
                         stop_cloudflared_container()
@@ -713,6 +722,13 @@ def settings_page():
         settings_form.preserve_unmanaged_cf_ingress_fields.data = current_app.config.get('PRESERVE_UNMANAGED_CF_INGRESS_FIELDS', False)
         settings_form.dockflare_public_url.data = current_app.config.get('DOCKFLARE_PUBLIC_URL', '')
         security_settings_form.disable_password_login.data = current_app.config.get('DISABLE_PASSWORD_LOGIN', False)
+        notification_config = current_app.config.get('NOTIFICATION_CONFIG', {})
+        notification_settings_form.enabled.data = bool(notification_config.get('enabled', False))
+        notification_settings_form.failure_cooldown_seconds.data = notification_config.get('failure_cooldown_seconds', 900)
+        for event_key, enabled in notification_config.get('events', {}).items():
+            field = getattr(notification_settings_form, event_key, None)
+            if field is not None:
+                field.data = bool(enabled)
 
     template_tunnel_state = {}
     template_agent_state = {}
@@ -731,6 +747,8 @@ def settings_page():
         change_password_form=change_password_form,
         security_settings_form=security_settings_form,
         cf_credentials_form=cf_credentials_form,
+        notification_settings_form=notification_settings_form,
+        notification_status=notification_manager.get_public_status(),
         all_account_tunnels=all_account_tunnels_list,
         tunnel_state=template_tunnel_state,
         agent_state=template_agent_state,
@@ -743,6 +761,102 @@ def settings_page():
         ACCOUNT_ID_FOR_DISPLAY=cf_account_id if cf_account_id else "Not Configured"
     )
 
+
+@bp.route('/settings/notifications', methods=['POST'])
+@login_required
+@limiter.limit("10 per minute")
+def save_notification_settings():
+    form = NotificationSettingsForm(prefix='notifications')
+    if not form.validate_on_submit():
+        flash(_t('flash.notification_settings_invalid'), 'danger')
+        return redirect(url_for('web.settings_page') + '#notifications')
+
+    config_data, fernet = config_loader.load_encrypted_config_with_cipher()
+    if config_data is None or fernet is None:
+        flash(_t('flash.notification_settings_save_failed'), 'danger')
+        return redirect(url_for('web.settings_page') + '#notifications')
+
+    from app.core.notification_manager import normalize_notification_config
+
+    previous_raw = config_data.get('notification_config')
+    previous_raw = copy.deepcopy(previous_raw) if isinstance(previous_raw, dict) else {}
+    previous = normalize_notification_config(previous_raw)
+    replacement_lines = [line.strip() for line in (form.replacement_urls.data or '').splitlines() if line.strip()]
+
+    if form.clear_urls.data:
+        urls = []
+    elif replacement_lines:
+        valid, invalid = notification_manager.validate_urls(replacement_lines)
+        if not valid:
+            line_number, scheme = invalid[0]
+            safe_detail = f"line {line_number}" if line_number else "destination list"
+            if scheme not in {'unknown', 'invalid', 'unavailable'}:
+                safe_detail += f" ({scheme})"
+            flash(_t('flash.notification_destination_invalid', detail=safe_detail), 'danger')
+            return redirect(url_for('web.settings_page') + '#notifications')
+        urls = normalize_notification_config({'urls': replacement_lines})['urls']
+    else:
+        urls = previous['urls']
+
+    if form.enabled.data and not urls:
+        flash(_t('flash.notification_destination_required'), 'danger')
+        return redirect(url_for('web.settings_page') + '#notifications')
+
+    stored_events = previous_raw.get('events')
+    stored_events = copy.deepcopy(stored_events) if isinstance(stored_events, dict) else {}
+    for event_key in previous['events']:
+        stored_events[event_key] = bool(getattr(form, event_key).data)
+
+    updated_raw = previous_raw
+    updated_raw.update({
+        'enabled': bool(form.enabled.data),
+        'urls': urls,
+        'events': stored_events,
+        'failure_cooldown_seconds': int(form.failure_cooldown_seconds.data),
+    })
+    config_data['notification_config'] = updated_raw
+    if not config_loader.save_encrypted_config(config_data, fernet):
+        flash(_t('flash.notification_settings_save_failed'), 'danger')
+        return redirect(url_for('web.settings_page') + '#notifications')
+
+    config_loader.apply_config_to_app(current_app, config_data)
+    notification_manager.configure(current_app.config['NOTIFICATION_CONFIG'])
+    flash(_t('flash.notification_settings_saved'), 'success')
+    return redirect(url_for('web.settings_page') + '#notifications')
+
+
+@bp.route('/api/v2/notifications/test', methods=['POST'])
+@login_required
+@limiter.limit("5 per minute")
+def test_notification_delivery():
+    status = notification_manager.get_public_status()
+    if not status['enabled'] or not status['available']:
+        return jsonify({'status': 'error', 'message': 'notifications_not_available'}), 409
+    job_id, accepted = notification_manager.send_test()
+    if not accepted:
+        return jsonify({'status': 'error', 'message': 'notification_queue_unavailable'}), 503
+    return jsonify({'status': 'accepted', 'job_id': job_id}), 202
+
+
+@bp.route('/api/v2/notifications/test/<job_id>', methods=['GET'])
+@login_required
+def notification_test_status(job_id):
+    job = notification_manager.get_test_status(job_id)
+    if not job:
+        return jsonify({'status': 'not_found'}), 404
+    response = {'status': job['status']}
+    if job.get('completed_at'):
+        response['completed_at'] = job['completed_at']
+    if job['status'] == 'failure':
+        response['message'] = 'Notification delivery failed. Check DockFlare logs.'
+    return jsonify(response), 200
+
+
+@bp.route('/api/v2/notifications/status', methods=['GET'])
+@login_required
+def notification_public_status():
+    return jsonify(notification_manager.get_public_status()), 200
+
 @bp.route('/settings/reveal-master-key', methods=['POST'])
 @login_required
 def reveal_master_key():
@@ -1131,12 +1245,14 @@ def reconciliation_status_route():
 
 @bp.route('/start-tunnel', methods=['POST'])
 def start_tunnel_route(): 
+    notification_manager.suppress_tunnel_health(tunnel_state.get("id"), 120)
     start_cloudflared_container() 
     time.sleep(1)
     return redirect(url_for('web.status_page'))
 
 @bp.route('/stop-tunnel', methods=['POST'])
 def stop_tunnel_route(): 
+    notification_manager.suppress_tunnel_health(tunnel_state.get("id"), 300)
     stop_cloudflared_container() 
     time.sleep(1)
     return redirect(url_for('web.status_page'))
@@ -1147,6 +1263,7 @@ def force_delete_rule_route(hostname):
     zone_id_for_delete = None
     access_app_id_for_delete = None
     tunnel_id_for_delete = None
+    source_for_delete = None
     fqdn = hostname.split('|')[0]
     with state_lock:
         rule_details = managed_rules.get(hostname)
@@ -1154,6 +1271,7 @@ def force_delete_rule_route(hostname):
             fqdn = rule_details.get("hostname") or fqdn
             zone_id_for_delete = rule_details.get("zone_id")
             access_app_id_for_delete = rule_details.get("access_app_id")
+            source_for_delete = rule_details.get("source") or "manual"
             tunnel_id_for_delete = rule_details.get("tunnel_id") or (tunnel_state.get("id") if not config.USE_EXTERNAL_CLOUDFLARED else config.EXTERNAL_TUNNEL_ID)
             del managed_rules[hostname]
             rule_removed_from_state = True
@@ -1166,12 +1284,29 @@ def force_delete_rule_route(hostname):
             and (rule.get("tunnel_id") or tunnel_id_for_delete) == tunnel_id_for_delete
             for rule in managed_rules.values()
         )
+    dns_deleted = True
+    access_deleted = True
+    tunnel_updated = True
     if zone_id_for_delete and tunnel_id_for_delete and not tuple_still_owned:
-        delete_cloudflare_dns_record(zone_id_for_delete, fqdn, tunnel_id_for_delete)
+        dns_deleted = bool(delete_cloudflare_dns_record(zone_id_for_delete, fqdn, tunnel_id_for_delete))
     if access_app_id_for_delete:
-        delete_cloudflare_access_application(access_app_id_for_delete)
+        access_deleted = bool(delete_cloudflare_access_application(access_app_id_for_delete))
     if rule_removed_from_state and not config.USE_EXTERNAL_CLOUDFLARED:
-        update_cloudflare_config(tunnel_id_for_delete)
+        tunnel_updated = bool(update_cloudflare_config(tunnel_id_for_delete))
+    if rule_removed_from_state:
+        context = {"hostname": fqdn, "operation": "force delete", "source": source_for_delete}
+        if not dns_deleted:
+            notification_manager.emit("cloudflare.dns_failure", f"{zone_id_for_delete}:{fqdn}", context)
+        if not access_deleted:
+            notification_manager.emit("cloudflare.access_failure", hostname, context)
+        if not tunnel_updated:
+            notification_manager.emit("cloudflare.tunnel_failure", str(tunnel_id_for_delete), context)
+        if dns_deleted and access_deleted and tunnel_updated:
+            notification_manager.emit(
+                "rule.deleted",
+                hostname,
+                {"source": source_for_delete, "resources": [{"key": hostname, "hostname": fqdn, "source": source_for_delete}], "public_url": config.DOCKFLARE_PUBLIC_URL},
+            )
     return redirect(url_for('web.status_page'))
 
 @bp.route('/stream-logs')

dockflare/package-lock.json

@@ -13,7 +13,7 @@
       "devDependencies": {
         "autoprefixer": "^10.4.19",
         "daisyui": "^4.10.2",
-        "postcss": "^8.4.38",
+        "postcss": "^8.5.26",
         "tailwindcss": "^3.4.17"
       }
     },
@@ -49,9 +49,9 @@
       }
     },
     "node_modules/@jridgewell/sourcemap-codec": {
-      "version": "1.5.5",
-      "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz",
-      "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==",
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz",
+      "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==",
       "license": "MIT"
     },
     "node_modules/@jridgewell/trace-mapping": {
@@ -174,9 +174,9 @@
       }
     },
     "node_modules/baseline-browser-mapping": {
-      "version": "2.11.7",
-      "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.7.tgz",
-      "integrity": "sha512-APw5YuIQAg6L9w4sHDI6j26DGFJI6RpYOhnkMPdC9lWbkKvsyPHzDsve1yd73lk21yz7Y09Kci8B2Pp9FonzWA==",
+      "version": "2.11.20",
+      "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.20.tgz",
+      "integrity": "sha512-H0ulySigv6icDJ1F7SjtdCD6PrhTpdYCmP0CactWy1+ekh0AFd0o1Wn5T8b+hnTmdBx19u9yhL6wvCylXMY7zw==",
       "dev": true,
       "license": "Apache-2.0",
       "bin": {
@@ -211,9 +211,9 @@
       }
     },
     "node_modules/browserslist": {
-      "version": "4.28.7",
-      "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.7.tgz",
-      "integrity": "sha512-JxV13hNrFxqjOc8alRbq9dK1MM79NEXYpma2B2J4wAtpWS5zIEIKqWPGCl7N4o7Uc7B7itylh7SuDujATRyyTw==",
+      "version": "4.28.8",
+      "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.8.tgz",
+      "integrity": "sha512-V2NpofLblG64mfOtSgDhOJESZEGogzDMBv/q+W6oc4LXWP/q75eOXoOaaOu1EOadB9U4Bwx/e0yzbvwKH8zalA==",
       "dev": true,
       "funding": [
         {
@@ -231,11 +231,11 @@
       ],
       "license": "MIT",
       "dependencies": {
-        "baseline-browser-mapping": "^2.10.44",
-        "caniuse-lite": "^1.0.30001806",
-        "electron-to-chromium": "^1.5.393",
-        "node-releases": "^2.0.51",
-        "update-browserslist-db": "^1.2.3"
+        "baseline-browser-mapping": "^2.11.12",
+        "caniuse-lite": "^1.0.30001809",
+        "electron-to-chromium": "^1.5.402",
+        "node-releases": "^2.0.53",
+        "update-browserslist-db": "^1.3.0"
       },
       "bin": {
         "browserslist": "cli.js"
@@ -254,9 +254,9 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001806",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001806.tgz",
-      "integrity": "sha512-72Cuvd95zbSYPKq6Fhg8eDJRlzgWDf7/mtoZv6Qe/DYNCEBdNxoA3+rZAU2ZhGCpZlns3EssFavaZomckT5Uuw==",
+      "version": "1.0.30001810",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz",
+      "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==",
       "dev": true,
       "funding": [
         {
@@ -385,9 +385,9 @@
       "license": "MIT"
     },
     "node_modules/electron-to-chromium": {
-      "version": "1.5.398",
-      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.398.tgz",
-      "integrity": "sha512-AsvhAxopJGh6museTDMIjn6JpDYOfgu4RLlygomt87MUwBUqTfd/1EiPtx10/LZE8xpTvkP2E9Gafq7lkLtodQ==",
+      "version": "1.5.416",
+      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.416.tgz",
+      "integrity": "sha512-K6bvB2BjnNrugtIih6ewlbBI9DXa976jIdiIlRLHhBoEI9a4JaQjjHyF+A1IQI543aQYR4LnmOrT/K5fZj0aPA==",
       "dev": true,
       "license": "ISC"
     },
@@ -446,9 +446,9 @@
       "license": "MIT"
     },
     "node_modules/fastq": {
-      "version": "1.20.1",
-      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz",
-      "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==",
+      "version": "1.20.2",
+      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.2.tgz",
+      "integrity": "sha512-UpGiiODyCGprM8EPP6JodP6jC9Rws6TCuiDOD+nn0CJhR8guI3g/ozo4ugL0vJ+Yz1UtJuuRPqvQuybVOF1VQA==",
       "license": "ISC",
       "dependencies": {
         "reusify": "^1.0.4"
@@ -680,9 +680,9 @@
       }
     },
     "node_modules/node-releases": {
-      "version": "2.0.51",
-      "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.51.tgz",
-      "integrity": "sha512-wRNIrw4DmVLKQlbgOMdkMx27Wrpzes2hh5Jtbi2bjPd+4wJstWIqP5A+lscnqbm0xxmT5Bpg8Lec5ItEBwx6BQ==",
+      "version": "2.0.54",
+      "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.54.tgz",
+      "integrity": "sha512-YHs7BmmcsdAI5Ozuf8JZo6PT0mv2GIWC9vMfvUC3dp65M8hn7Ux8CPL+2oBI7juNuj9d0ndhTcznq2ODBps9cQ==",
       "dev": true,
       "license": "MIT",
       "engines": {
@@ -740,15 +740,6 @@
         "url": "https://github.com/sponsors/jonschlinkert"
       }
     },
-    "node_modules/pify": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
-      "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/pirates": {
       "version": "4.0.7",
       "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz",
@@ -759,9 +750,9 @@
       }
     },
     "node_modules/postcss": {
-      "version": "8.5.25",
-      "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz",
-      "integrity": "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==",
+      "version": "8.5.26",
+      "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz",
+      "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==",
       "funding": [
         {
           "type": "opencollective",
@@ -778,7 +769,7 @@
       ],
       "license": "MIT",
       "dependencies": {
-        "nanoid": "^3.3.16",
+        "nanoid": "^3.3.17",
         "picocolors": "^1.1.1",
         "source-map-js": "^1.2.1"
       },
@@ -948,13 +939,10 @@
       "license": "MIT"
     },
     "node_modules/read-cache": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz",
-      "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==",
-      "license": "MIT",
-      "dependencies": {
-        "pify": "^2.3.0"
-      }
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.2.tgz",
+      "integrity": "sha512-/peqiBB/n07gQGLsWaHho3WfvUyRscw0gYTsEFMhrIe/nWLkYaf5SbKYjGYqtRV3aPwykJgF2VEMo1ac4bnsGA==",
+      "license": "MIT"
     },
     "node_modules/readdirp": {
       "version": "3.6.0",
@@ -1171,9 +1159,9 @@
       "license": "Apache-2.0"
     },
     "node_modules/update-browserslist-db": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz",
-      "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==",
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.2.tgz",
+      "integrity": "sha512-UQ+MSxlhRm1bzjhU+DcuXfjFO1FzNtqhK5+9Yvlp90ItDLk5vT932A0rFu619nf7RVS+Y/VeaUW1jaRDqZ8VJw==",
       "dev": true,
       "funding": [
         {

dockflare/package.json

@@ -13,7 +13,7 @@
   "devDependencies": {
     "autoprefixer": "^10.4.19",
     "daisyui": "^4.10.2",
-    "postcss": "^8.4.38",
+    "postcss": "^8.5.26",
     "tailwindcss": "^3.4.17"
   },
   "dependencies": {

dockflare/requirements.in

@@ -1,5 +1,5 @@
 Authlib==1.6.11
-boto3
+boto3==1.42.83
 cryptography==46.0.7
 docker==7.1.0
 Flask==3.1.3
@@ -8,6 +8,7 @@ Flask-Limiter==3.13
 Flask-Login==0.6.3
 Flask-WTF==1.2.2
 Markdown==3.8.1
+apprise==1.13.1
 Pygments==2.20.0
 PyJWT[crypto]>=2.8.0
 pymdown-extensions==10.21.2

dockflare/requirements.txt

@@ -4,34 +4,37 @@
 #
 #    pip-compile --output-file=dockflare/requirements.txt dockflare/requirements.in
 #
+apprise==1.13.1
+    # via -r dockflare/requirements.in
 authlib==1.6.11
     # via -r dockflare/requirements.in
+blinker==1.9.0
+    # via flask
 boto3==1.42.83
     # via -r dockflare/requirements.in
 botocore==1.42.88
-    # via boto3
-jmespath==1.1.0
-    # via boto3, botocore
-python-dateutil==2.9.0.post0
-    # via botocore
-s3transfer==0.16.0
-    # via boto3
-blinker==1.9.0
-    # via flask
+    # via
+    #   boto3
+    #   s3transfer
 cachelib==0.9.0
     # via flask-caching
 certifi==2025.8.3
-    # via requests
+    # via
+    #   apprise
+    #   requests
 cffi==2.0.0
     # via cryptography
 charset-normalizer==3.4.2
     # via requests
 click==8.2.1
-    # via flask
+    # via
+    #   apprise
+    #   flask
 cryptography==46.0.7
     # via
     #   -r dockflare/requirements.in
     #   authlib
+    #   pyjwt
 deprecated==1.3.1
     # via limits
 docker==7.1.0
@@ -59,11 +62,16 @@ itsdangerous==2.2.0
     #   flask-wtf
 jinja2==3.1.6
     # via flask
+jmespath==1.1.0
+    # via
+    #   boto3
+    #   botocore
 limits==5.8.0
     # via flask-limiter
 markdown==3.8.1
     # via
     #   -r dockflare/requirements.in
+    #   apprise
     #   pymdown-extensions
 markdown-it-py==4.0.0
     # via rich
@@ -75,6 +83,8 @@ markupsafe==3.0.2
     #   wtforms
 mdurl==0.1.2
     # via markdown-it-py
+oauthlib==3.3.1
+    # via requests-oauthlib
 ordered-set==4.1.0
     # via flask-limiter
 packaging==26.0
@@ -85,20 +95,36 @@ pygments==2.20.0
     # via
     #   -r dockflare/requirements.in
     #   rich
+pyjwt[crypto]==2.12.1
+    # via
+    #   -r dockflare/requirements.in
+    #   pyjwt
 pymdown-extensions==10.21.2
     # via -r dockflare/requirements.in
+python-dateutil==2.9.0.post0
+    # via botocore
 python-dotenv==1.2.2
     # via -r dockflare/requirements.in
 pyyaml==6.0.3
-    # via pymdown-extensions
+    # via
+    #   apprise
+    #   pymdown-extensions
 redis==4.5.5
     # via -r dockflare/requirements.in
 requests==2.33.1
     # via
     #   -r dockflare/requirements.in
+    #   apprise
     #   docker
+    #   requests-oauthlib
+requests-oauthlib==2.0.0
+    # via apprise
 rich==14.3.3
     # via flask-limiter
+s3transfer==0.16.0
+    # via boto3
+six==1.17.0
+    # via python-dateutil
 typing-extensions==4.15.0
     # via
     #   flask-limiter
@@ -106,6 +132,7 @@ typing-extensions==4.15.0
 urllib3==2.6.3
     # via
     #   -r dockflare/requirements.in
+    #   botocore
     #   docker
     #   requests
 waitress==3.0.2
@@ -119,5 +146,3 @@ wrapt==2.1.2
     # via deprecated
 wtforms==3.2.1
     # via flask-wtf
-pyjwt[crypto]==2.12.1
-    # via -r dockflare/requirements.in

dockflare/tests/test_notification_events.py

@@ -0,0 +1,112 @@
+import copy
+import unittest
+from datetime import datetime, timedelta, timezone
+from unittest.mock import patch
+
+from app.core import docker_handler, reconciler
+from app.core.state_manager import managed_rules, state_lock
+
+
+class _Container:
+    id = "notification-container"
+    name = "notification-app"
+    labels = {
+        "dockflare.enable": "true",
+        "dockflare.hostname": "notify.example.com",
+        "dockflare.service": "http://notification-app:8080",
+    }
+
+    def reload(self):
+        return None
+
+
+class NotificationEventBoundaryTests(unittest.TestCase):
+    def setUp(self):
+        with state_lock:
+            self.rules_snapshot = copy.deepcopy(managed_rules)
+            managed_rules.clear()
+
+    def tearDown(self):
+        with state_lock:
+            managed_rules.clear()
+            managed_rules.update(self.rules_snapshot)
+
+    @staticmethod
+    def _inventory():
+        return {
+            "zones": [{"id": "zone-1", "name": "example.com"}],
+            "status": "complete",
+            "error": None,
+        }
+
+    def _process_start(self, tunnel_success=True):
+        emitted = []
+
+        def capture(event_type, resource_id, context=None, notify_type=None):
+            if hasattr(state_lock, "_is_owned"):
+                self.assertFalse(state_lock._is_owned())
+            emitted.append((event_type, resource_id, context or {}))
+            return True
+
+        with patch.object(docker_handler, "get_account_zone_inventory", return_value=self._inventory()), \
+             patch.object(docker_handler, "save_state", return_value=True), \
+             patch.object(docker_handler, "publish_state_event"), \
+             patch.object(docker_handler, "handle_access_policy_from_labels", return_value=False), \
+             patch.object(docker_handler, "update_cloudflare_config", return_value=tunnel_success), \
+             patch.object(docker_handler, "create_cloudflare_dns_record", return_value="dns-id"), \
+             patch.object(docker_handler.notification_manager, "emit", side_effect=capture), \
+             patch.dict(docker_handler.tunnel_state, {"id": "master-tunnel", "name": "Master"}, clear=False):
+            docker_handler.process_container_start(_Container())
+        return emitted
+
+    def test_new_rule_emits_activation_only_after_remote_success(self):
+        emitted = self._process_start(tunnel_success=True)
+        self.assertEqual([event[0] for event in emitted].count("rule.activated"), 1)
+        self.assertNotIn("cloudflare.tunnel_failure", [event[0] for event in emitted])
+
+    def test_tunnel_failure_emits_failure_and_not_activation(self):
+        emitted = self._process_start(tunnel_success=False)
+        event_types = [event[0] for event in emitted]
+        self.assertIn("cloudflare.tunnel_failure", event_types)
+        self.assertNotIn("rule.activated", event_types)
+
+    def test_cleanup_notifies_only_committed_rules(self):
+        now = datetime(2026, 8, 31, 12, 0, tzinfo=timezone.utc)
+        with state_lock:
+            for name, tunnel_id in (("deleted", "tunnel-ok"), ("retained", "tunnel-failed")):
+                managed_rules[f"{name}.example.com|"] = {
+                    "hostname": f"{name}.example.com",
+                    "zone_id": "zone-1",
+                    "status": "pending_deletion",
+                    "delete_at": now - timedelta(seconds=1),
+                    "source": "docker",
+                    "container_id": name,
+                    "tunnel_id": tunnel_id,
+                    "lifecycle_generation": 1,
+                }
+        emitted = []
+        with patch.object(
+            reconciler, "update_cloudflare_config",
+            side_effect=lambda tunnel_id: tunnel_id == "tunnel-ok",
+        ), patch.object(reconciler, "delete_cloudflare_dns_record", return_value=True), \
+             patch.object(reconciler, "save_state", return_value=True), \
+             patch.object(reconciler, "publish_state_event"), \
+             patch.object(
+                 reconciler.notification_manager, "emit",
+                 side_effect=lambda event_type, resource_id, context=None, notify_type=None:
+                 emitted.append((event_type, context or {})),
+             ):
+            committed = reconciler.cleanup_expired_rules_once(now)
+
+        self.assertEqual(committed, 1)
+        deleted_event = next(context for event_type, context in emitted if event_type == "rule.deleted")
+        self.assertEqual(
+            [resource["hostname"] for resource in deleted_event["resources"]],
+            ["deleted.example.com"],
+        )
+        self.assertIn("cloudflare.tunnel_failure", [event_type for event_type, _ in emitted])
+        self.assertIn("retained.example.com|", managed_rules)
+
+
+if __name__ == "__main__":
+    unittest.main()

dockflare/tests/test_notification_manager.py

@@ -0,0 +1,252 @@
+import logging
+import threading
+import unittest
+from datetime import datetime, timedelta, timezone
+from unittest.mock import patch
+
+from app.core.notification_manager import (
+    DEFAULT_NOTIFICATION_CONFIG,
+    NotificationManager,
+    normalize_notification_config,
+    redact_destination,
+    sanitize_service,
+)
+
+
+class FakeNotifyType:
+    INFO = "info"
+    SUCCESS = "success"
+    WARNING = "warning"
+    FAILURE = "failure"
+
+
+class FakeAppriseAsset:
+    def __init__(self, **kwargs):
+        self.kwargs = kwargs
+
+
+class FakeApprise:
+    instances = []
+    result = True
+    error = None
+
+    def __init__(self, asset=None):
+        self.asset = asset
+        self.urls = []
+        self.calls = []
+        self.__class__.instances.append(self)
+
+    def add(self, url):
+        if url.startswith("invalid://"):
+            return False
+        self.urls.append(url)
+        return True
+
+    def notify(self, **kwargs):
+        self.calls.append(kwargs)
+        if self.__class__.error:
+            raise self.__class__.error
+        return self.__class__.result
+
+    def __len__(self):
+        return len(self.urls)
+
+
+class FakeAppriseModule:
+    Apprise = FakeApprise
+    AppriseAsset = FakeAppriseAsset
+    NotifyType = FakeNotifyType
+
+
+def enabled_config(**overrides):
+    value = {
+        "enabled": True,
+        "urls": ["json://example.test/path?token=super-secret"],
+        "events": dict(DEFAULT_NOTIFICATION_CONFIG["events"]),
+        "failure_cooldown_seconds": 900,
+    }
+    value["events"].update({key: True for key in value["events"]})
+    value.update(overrides)
+    return value
+
+
+class NotificationConfigurationTests(unittest.TestCase):
+    def setUp(self):
+        FakeApprise.instances.clear()
+        FakeApprise.result = True
+        FakeApprise.error = None
+
+    def test_defaults_are_disabled_and_failure_events_default_on(self):
+        config = normalize_notification_config(None)
+        self.assertFalse(config["enabled"])
+        self.assertTrue(config["events"]["cloudflare_dns_failure"])
+        self.assertFalse(config["events"]["rule_activated"])
+        self.assertEqual(config["failure_cooldown_seconds"], 900)
+
+    def test_urls_are_trimmed_deduplicated_and_limited(self):
+        urls = [" json://one.test ", "json://one.test"] + [f"json://{index}.test" for index in range(40)]
+        config = normalize_notification_config({"urls": urls})
+        self.assertEqual(config["urls"][0], "json://one.test")
+        self.assertEqual(len(config["urls"]), 32)
+
+    def test_invalid_cooldown_uses_default(self):
+        self.assertEqual(normalize_notification_config({"failure_cooldown_seconds": 59})["failure_cooldown_seconds"], 900)
+        self.assertEqual(normalize_notification_config({"failure_cooldown_seconds": True})["failure_cooldown_seconds"], 900)
+
+    def test_destination_and_service_redaction_remove_secrets(self):
+        secret_url = "discord://user:password@example.test/webhook-token?token=query-secret#fragment"
+        self.assertEqual(redact_destination(secret_url), "discord://configured destination")
+        sanitized = sanitize_service("https://user:password@example.test:8443/path?token=query-secret#fragment")
+        self.assertEqual(sanitized, "https://example.test:8443/path")
+        self.assertNotIn("password", sanitized)
+        self.assertNotIn("query-secret", sanitized)
+
+    def test_validation_reports_only_line_and_scheme(self):
+        manager = NotificationManager()
+        with patch.object(manager, "_load_apprise", return_value=FakeAppriseModule):
+            valid, invalid = manager.validate_urls(["invalid://do-not-echo-secret"])
+        self.assertFalse(valid)
+        self.assertEqual(invalid, [(1, "invalid")])
+
+
+class NotificationManagerTests(unittest.TestCase):
+    def setUp(self):
+        FakeApprise.instances.clear()
+        FakeApprise.result = True
+        FakeApprise.error = None
+        self.manager = NotificationManager(queue_size=4, max_body_chars=512, resource_limit=2)
+        self.apprise_patch = patch.object(self.manager, "_load_apprise", return_value=FakeAppriseModule)
+        self.apprise_patch.start()
+        self.manager.configure(enabled_config())
+        self.manager.end_bootstrap()
+
+    def tearDown(self):
+        self.manager.stop(0.1)
+        self.apprise_patch.stop()
+
+    def test_configure_exposes_redacted_status_only(self):
+        status = self.manager.get_public_status()
+        serialized = str(status)
+        self.assertTrue(status["available"])
+        self.assertEqual(status["configured_destination_count"], 1)
+        self.assertNotIn("super-secret", serialized)
+        self.assertEqual(status["destinations"][0]["summary"], "json://configured destination")
+
+    def test_worker_delivers_mapped_type_and_truncated_resource_list(self):
+        stop_event = threading.Event()
+        self.manager.start(stop_event)
+        accepted = self.manager.emit(
+            "rule.activated",
+            "container-a",
+            {
+                "resources": [
+                    {"hostname": "one.example.test"},
+                    {"hostname": "two.example.test"},
+                    {"hostname": "three.example.test"},
+                ]
+            },
+        )
+        self.assertTrue(accepted)
+        self.manager._queue.join()
+        call = FakeApprise.instances[-1].calls[-1]
+        self.assertEqual(call["notify_type"], "success")
+        self.assertIn("one.example.test", call["body"])
+        self.assertIn("and 1 more", call["body"])
+        self.assertNotIn("three.example.test", call["body"])
+        stop_event.set()
+
+    def test_bootstrap_suppresses_routine_events_but_allows_failures(self):
+        self.manager.begin_bootstrap()
+        self.assertFalse(self.manager.emit("rule.activated", "rule-a", {}))
+        self.assertTrue(self.manager.emit("cloudflare.dns_failure", "rule-a", {"hostname": "a.example.test"}))
+
+    def test_disabled_and_event_preferences_suppress_before_queueing(self):
+        self.manager.configure(enabled_config(enabled=False))
+        self.assertFalse(self.manager.emit("cloudflare.dns_failure", "dns-a", {}))
+        self.manager.configure(enabled_config(events={key: False for key in DEFAULT_NOTIFICATION_CONFIG["events"]}))
+        self.assertFalse(self.manager.emit("cloudflare.dns_failure", "dns-a", {}))
+        self.assertEqual(self.manager.get_public_status()["queue_depth"], 0)
+
+    def test_failure_cooldown_and_correlated_recovery(self):
+        self.assertTrue(self.manager.emit("agent.offline", "agent-a", {}))
+        self.assertFalse(self.manager.emit("agent.offline", "agent-a", {}))
+        self.assertTrue(self.manager.emit("agent.online", "agent-a", {}))
+        self.assertFalse(self.manager.emit("agent.online", "agent-a", {}))
+        self.assertTrue(self.manager.emit("agent.offline", "agent-a", {}))
+
+    def test_queue_full_drops_without_blocking(self):
+        for index in range(4):
+            self.assertTrue(self.manager.emit("rule.activated", f"rule-{index}", {}))
+        self.assertFalse(self.manager.emit("rule.activated", "rule-overflow", {}))
+        self.assertEqual(self.manager.get_public_status()["dropped_events"], 1)
+
+    def test_test_job_bypasses_event_preferences(self):
+        self.manager.configure(enabled_config(events={key: False for key in DEFAULT_NOTIFICATION_CONFIG["events"]}))
+        job_id, accepted = self.manager.send_test()
+        self.assertTrue(accepted)
+        self.assertEqual(self.manager.get_test_status(job_id)["status"], "pending")
+
+    def test_delivery_exception_does_not_log_secret_exception_text(self):
+        FakeApprise.error = RuntimeError("super-secret upstream response")
+        stop_event = threading.Event()
+        with self.assertLogs(level=logging.ERROR) as captured:
+            self.manager.start(stop_event)
+            self.assertTrue(self.manager.emit("cloudflare.dns_failure", "dns-a", {}))
+            self.manager._queue.join()
+        output = "\n".join(captured.output)
+        self.assertIn("RuntimeError", output)
+        self.assertNotIn("super-secret upstream response", output)
+        stop_event.set()
+
+    def test_worker_continues_after_delivery_exception(self):
+        stop_event = threading.Event()
+        self.manager.start(stop_event)
+        FakeApprise.error = RuntimeError("first delivery fails")
+        self.assertTrue(self.manager.emit("cloudflare.dns_failure", "dns-a", {}))
+        self.manager._queue.join()
+        FakeApprise.error = None
+        self.assertTrue(self.manager.emit("cloudflare.dns_failure", "dns-b", {}))
+        self.manager._queue.join()
+        self.assertEqual(len(FakeApprise.instances[-1].calls), 2)
+        self.assertIsNotNone(self.manager.get_public_status()["last_success_at"])
+        stop_event.set()
+
+    def test_agent_health_establishes_baseline_then_emits_outage_and_recovery(self):
+        events = []
+        self.manager.emit = lambda event_type, resource_id, context=None, notify_type=None: events.append((event_type, resource_id)) or True
+        now = datetime.now(timezone.utc)
+        online = {"agent-a": {"status": "enrolled", "display_name": "A", "last_seen": now.isoformat()}}
+        offline = {"agent-a": {"status": "enrolled", "display_name": "A", "last_seen": (now - timedelta(minutes=5)).isoformat()}}
+        self.manager.check_agent_health(online, 60, now=now)
+        self.manager.check_agent_health(offline, 60, now=now)
+        self.manager.check_agent_health(online, 60, now=now)
+        self.assertEqual(events, [("agent.offline", "agent-a"), ("agent.online", "agent-a")])
+
+    def test_tunnel_health_requires_two_failed_checks(self):
+        events = []
+        self.manager.emit = lambda event_type, resource_id, context=None, notify_type=None: events.append((event_type, resource_id)) or True
+        self.manager.check_tunnel_health("tunnel-a", "running")
+        self.manager.check_tunnel_health("tunnel-a", "exited")
+        self.assertEqual(events, [])
+        self.manager.check_tunnel_health("tunnel-a", "exited")
+        self.manager.check_tunnel_health("tunnel-a", "running")
+        self.assertEqual(events, [("tunnel.down", "tunnel-a"), ("tunnel.recovered", "tunnel-a")])
+
+    def test_intentional_tunnel_stop_and_agent_decommission_are_suppressed(self):
+        events = []
+        self.manager.emit = lambda event_type, resource_id, context=None, notify_type=None: events.append((event_type, resource_id)) or True
+        self.manager.check_tunnel_health("tunnel-a", "running")
+        self.manager.suppress_tunnel_health("tunnel-a", 60)
+        self.manager.check_tunnel_health("tunnel-a", "exited")
+        self.manager.check_tunnel_health("tunnel-a", "exited")
+
+        now = datetime.now(timezone.utc)
+        online = {"agent-a": {"status": "enrolled", "last_seen": now.isoformat()}}
+        offline = {"agent-a": {"status": "enrolled", "last_seen": (now - timedelta(minutes=5)).isoformat()}}
+        self.manager.check_agent_health(online, 60, now=now)
+        self.manager.check_agent_health(offline, 60, now=now, decommissioning_ids={"agent-a"})
+        self.assertEqual(events, [])
+
+
+if __name__ == "__main__":
+    unittest.main()

dockflare/tests/test_notification_routes.py

@@ -0,0 +1,207 @@
+import copy
+import unittest
+from unittest.mock import patch
+
+from app import app
+from app.core.notification_manager import normalize_notification_config
+from app.web import routes
+
+
+class NotificationRouteTests(unittest.TestCase):
+    def setUp(self):
+        self.config_snapshot = {
+            key: copy.deepcopy(app.config.get(key))
+            for key in (
+                "TESTING", "WTF_CSRF_ENABLED", "DOCKFLARE_USERNAME",
+                "NOTIFICATION_CONFIG", "TUNNEL_DNS_SCAN_ZONE_NAMES",
+            )
+        }
+        self.configured_snapshot = getattr(app, "is_configured", False)
+        app.config.update(
+            TESTING=True,
+            WTF_CSRF_ENABLED=False,
+            DOCKFLARE_USERNAME="notification-admin",
+            NOTIFICATION_CONFIG=normalize_notification_config(None),
+            TUNNEL_DNS_SCAN_ZONE_NAMES=[],
+        )
+        app.is_configured = True
+        self.client = app.test_client()
+
+    def tearDown(self):
+        for key, value in self.config_snapshot.items():
+            if value is None:
+                app.config.pop(key, None)
+            else:
+                app.config[key] = value
+        app.is_configured = self.configured_snapshot
+
+    def _authenticate(self):
+        with self.client.session_transaction() as session:
+            session["_user_id"] = "notification-admin"
+            session["_fresh"] = True
+
+    @staticmethod
+    def _form(**overrides):
+        values = {
+            "notifications-failure_cooldown_seconds": "900",
+            "notifications-cloudflare_tunnel_failure": "y",
+            "notifications-cloudflare_dns_failure": "y",
+            "notifications-cloudflare_access_failure": "y",
+            "notifications-docker_listener_failure": "y",
+            "notifications-agent_offline": "y",
+            "notifications-agent_online": "y",
+            "notifications-tunnel_down": "y",
+            "notifications-tunnel_recovered": "y",
+        }
+        values.update(overrides)
+        return values
+
+    def test_notification_api_requires_login(self):
+        response = self.client.get("/api/v2/notifications/status")
+        self.assertEqual(response.status_code, 302)
+        self.assertIn("/login", response.headers["Location"])
+
+    def test_notification_write_requires_csrf(self):
+        self._authenticate()
+        app.config["WTF_CSRF_ENABLED"] = True
+        response = self.client.post("/api/v2/notifications/test", json={})
+        self.assertEqual(response.status_code, 400)
+
+    def test_settings_html_never_contains_saved_url(self):
+        self._authenticate()
+        secret = "sentinel-secret-must-not-reach-browser"
+        app.config["NOTIFICATION_CONFIG"] = normalize_notification_config({
+            "enabled": True,
+            "urls": [f"json://user:{secret}@notify.example/private?token={secret}"],
+        })
+        public_status = {
+            "enabled": True,
+            "available": True,
+            "configured_destination_count": 1,
+            "destinations": [{"summary": "json://configured destination"}],
+        }
+        with patch.object(routes, "get_all_account_cloudflare_tunnels", return_value=[]), \
+             patch.object(routes.notification_manager, "get_public_status", return_value=public_status):
+            response = self.client.get("/settings")
+        self.assertEqual(response.status_code, 200)
+        self.assertNotIn(secret.encode(), response.data)
+        self.assertNotIn(b"notify.example", response.data)
+
+    def test_blank_replacement_retains_urls_and_unknown_fields(self):
+        self._authenticate()
+        secret_url = "json://user:secret@example.invalid/path"
+        stored = {
+            "notification_config": {
+                "enabled": False,
+                "urls": [secret_url],
+                "events": {"future_event": True},
+                "failure_cooldown_seconds": 120,
+                "future_option": "keep-me",
+            },
+            "unrelated": "keep-this-too",
+        }
+
+        def apply_runtime(flask_app, config_data):
+            flask_app.config["NOTIFICATION_CONFIG"] = normalize_notification_config(
+                config_data["notification_config"]
+            )
+
+        with patch.object(
+            routes.config_loader, "load_encrypted_config_with_cipher",
+            return_value=(stored, object()),
+        ), patch.object(
+            routes.config_loader, "save_encrypted_config", return_value=True,
+        ) as save_config, patch.object(
+            routes.config_loader, "apply_config_to_app", side_effect=apply_runtime,
+        ), patch.object(routes.notification_manager, "configure") as configure:
+            response = self.client.post(
+                "/settings/notifications",
+                data=self._form(**{"notifications-enabled": "y"}),
+            )
+
+        self.assertEqual(response.status_code, 302)
+        saved = save_config.call_args.args[0]
+        self.assertEqual(saved["notification_config"]["urls"], [secret_url])
+        self.assertEqual(saved["notification_config"]["future_option"], "keep-me")
+        self.assertTrue(saved["notification_config"]["events"]["future_event"])
+        self.assertEqual(saved["unrelated"], "keep-this-too")
+        configure.assert_called_once_with(app.config["NOTIFICATION_CONFIG"])
+
+    def test_enabled_without_destination_is_rejected(self):
+        self._authenticate()
+        stored = {"notification_config": normalize_notification_config(None)}
+        with patch.object(
+            routes.config_loader, "load_encrypted_config_with_cipher",
+            return_value=(stored, object()),
+        ), patch.object(routes.config_loader, "save_encrypted_config") as save_config:
+            response = self.client.post(
+                "/settings/notifications",
+                data=self._form(**{"notifications-enabled": "y"}),
+            )
+        self.assertEqual(response.status_code, 302)
+        save_config.assert_not_called()
+
+    def test_invalid_replacement_does_not_persist_or_reload(self):
+        self._authenticate()
+        stored = {"notification_config": {"urls": ["json://saved.invalid/path"]}}
+        with patch.object(
+            routes.config_loader, "load_encrypted_config_with_cipher",
+            return_value=(stored, object()),
+        ), patch.object(
+            routes.notification_manager, "validate_urls",
+            return_value=(False, [(1, "discord")]),
+        ), patch.object(
+            routes.config_loader, "save_encrypted_config",
+        ) as save_config, patch.object(routes.notification_manager, "configure") as configure:
+            response = self.client.post(
+                "/settings/notifications",
+                data=self._form(**{
+                    "notifications-enabled": "y",
+                    "notifications-replacement_urls": "discord://sentinel-secret",
+                }),
+            )
+        self.assertEqual(response.status_code, 302)
+        save_config.assert_not_called()
+        configure.assert_not_called()
+
+    def test_test_endpoint_is_rate_limited(self):
+        self._authenticate()
+        status = {"enabled": True, "available": True}
+        with patch.object(routes.notification_manager, "get_public_status", return_value=status), \
+             patch.object(routes.notification_manager, "send_test", return_value=("job", True)):
+            responses = [
+                self.client.post(
+                    "/api/v2/notifications/test",
+                    json={},
+                    environ_overrides={"REMOTE_ADDR": "192.0.2.77"},
+                )
+                for _ in range(6)
+            ]
+        self.assertEqual([response.status_code for response in responses[:5]], [202] * 5)
+        self.assertEqual(responses[5].status_code, 429)
+
+    def test_test_job_routes_return_only_opaque_status(self):
+        self._authenticate()
+        status = {"enabled": True, "available": True}
+        with patch.object(routes.notification_manager, "get_public_status", return_value=status), \
+             patch.object(routes.notification_manager, "send_test", return_value=("opaque-job-id", True)):
+            response = self.client.post("/api/v2/notifications/test", json={})
+        self.assertEqual(response.status_code, 202)
+        self.assertEqual(response.get_json(), {"status": "accepted", "job_id": "opaque-job-id"})
+
+        with patch.object(routes.notification_manager, "get_test_status", return_value={
+            "status": "failure",
+            "completed_at": "2026-01-01T00:00:00+00:00",
+        }):
+            response = self.client.get("/api/v2/notifications/test/opaque-job-id")
+        self.assertEqual(response.status_code, 200)
+        self.assertNotIn("url", str(response.get_json()).lower())
+        self.assertEqual(response.get_json()["status"], "failure")
+
+        with patch.object(routes.notification_manager, "get_test_status", return_value=None):
+            response = self.client.get("/api/v2/notifications/test/missing")
+        self.assertEqual(response.status_code, 404)
+
+
+if __name__ == "__main__":
+    unittest.main()

webmail/package.json

@@ -21,17 +21,17 @@
     "@tiptap/extension-underline": "^2.27.2",
     "@tiptap/starter-kit": "^2.6.0",
     "@tiptap/vue-3": "^2.6.0",
-    "axios": "^1.15.0",
+    "axios": "^1.20.0",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.0",
     "date-fns": "^3.6.0",
-    "dompurify": "^3.4.0",
+    "dompurify": "^3.4.14",
     "emoji-mart": "^5.6.0",
     "lucide-vue-next": "^0.400.0",
     "pinia": "^2.2.0",
     "radix-vue": "^1.9.0",
     "tailwind-merge": "^2.5.0",
-    "vue": "^3.5.0",
+    "vue": "^3.5.42",
     "vue-router": "^4.4.0"
   },
   "overrides": {
@@ -42,7 +42,7 @@
   "devDependencies": {
     "@vitejs/plugin-vue": "^5.1.0",
     "autoprefixer": "^10.4.0",
-    "postcss": "^8.4.0",
+    "postcss": "^8.5.26",
     "tailwindcss": "^3.4.0",
     "typescript": "^5.5.0",
     "vite": "^5.4.0",