Murat ÖZDEMİR
03a3c5fb70
Adds two-layer BE-* microservice monitoring: an aggregate Stack Services replica check across the iklimco and monitoring Swarm stacks (checks/swarm_services.py), and per-service actuator/DNS health checks driven by a new microservice_monitors section in monitors.yml (checks/actuator.py). BE-* services are excluded from the Stack Services aggregate so a single service incident produces exactly one Slack alert instead of two. Fixes found during QA of the initial implementation: RabbitMQ host derivation generated nonexistent numbered DNS names instead of using the real single-service topology, RabbitMQ and Patroni cluster checks did not push down status on node/member shortfall, and the etcd cluster check still used a hardcoded node list and quorum threshold instead of deriving them from CLUSTER_SIZE_ETCD. Also hardens Docker event crash alerting: excludes ephemeral Gitea Actions runner containers from crash notifications, skips alerts for containers Swarm intentionally stopped (rolling update/scale-down, detected via task DesiredState) instead of alerting on every deploy, and adds human-readable exit code descriptions to the Slack message. Uptime Kuma monitor names are now prefixed with "iklim [env]" since test and prod share one Kuma instance and unprefixed names collided. setup_uptime_kuma.py gained an ensure_push_monitor helper and a microservice push-monitor creation pass. Removed state.py and the unused restart_threshold config field (dead code, no longer referenced anywhere). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
79 lines
3.1 KiB
Python
79 lines
3.1 KiB
Python
import time
|
|
import docker
|
|
import logging
|
|
from health_agent.uptime_kuma import push
|
|
from health_agent import config
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Both Swarm stacks running on the host: microservices + infra ("iklimco")
|
|
# and the monitoring stack itself ("monitoring", this agent's own stack).
|
|
STACK_NAMESPACES = ("iklimco", "monitoring")
|
|
|
|
def check_stack_services():
|
|
start_time = time.time()
|
|
try:
|
|
client = docker.from_env()
|
|
services = []
|
|
for namespace in STACK_NAMESPACES:
|
|
services.extend(client.services.list(filters={"label": f"com.docker.stack.namespace={namespace}"}))
|
|
|
|
# BE-* microservices are covered individually by the actuator checks
|
|
# (checks/actuator.py, driven by monitors.yml's microservice_monitors),
|
|
# which already detect missing/short replicas via DNS + health status.
|
|
# Excluded here so a single service incident doesn't also flip this
|
|
# aggregate down and fire a second, redundant notification.
|
|
microservice_names = {m['service'] for m in config.load_microservice_monitors()}
|
|
|
|
# Gather swarm services by short name, skipping microservice-covered ones
|
|
swarm_services_by_name = {}
|
|
for svc in services:
|
|
name = svc.name
|
|
short_name = name
|
|
for namespace in STACK_NAMESPACES:
|
|
prefix = f"{namespace}_"
|
|
if name.startswith(prefix):
|
|
short_name = name[len(prefix):]
|
|
break
|
|
if short_name in microservice_names:
|
|
continue
|
|
swarm_services_by_name[short_name] = svc
|
|
|
|
missing_or_down = []
|
|
total_services = 0
|
|
|
|
for short_name, svc in swarm_services_by_name.items():
|
|
spec = svc.attrs.get('Spec', {})
|
|
mode = spec.get('Mode', {})
|
|
expected = 0
|
|
|
|
if 'Replicated' in mode:
|
|
expected = mode['Replicated'].get('Replicas', 0)
|
|
elif 'Global' in mode:
|
|
nodes = client.nodes.list()
|
|
expected = sum(1 for n in nodes if n.attrs.get('Status', {}).get('State') == 'ready')
|
|
|
|
if expected == 0:
|
|
continue # Intentionally scaled down and no config expectation
|
|
|
|
total_services += 1
|
|
tasks = client.api.tasks(filters={"service": svc.id, "desired-state": "running"})
|
|
running_tasks = sum(1 for t in tasks if t.get('Status', {}).get('State') == 'running')
|
|
|
|
if running_tasks < expected:
|
|
missing_or_down.append(f"{short_name} {running_tasks}/{expected}")
|
|
|
|
ping_ms = int((time.time() - start_time) * 1000)
|
|
|
|
if not missing_or_down:
|
|
msg = f"{total_services} services, all replicas running"
|
|
push("Stack Services", "up", msg, ping_ms)
|
|
else:
|
|
msg = ", ".join(missing_or_down)
|
|
push("Stack Services", "down", msg, ping_ms)
|
|
|
|
except Exception as e:
|
|
ping_ms = int((time.time() - start_time) * 1000)
|
|
logger.error(f"Stack services check failed: {e}")
|
|
push("Stack Services", "down", str(e), ping_ms)
|