import time import docker import logging from health_agent.uptime_kuma import push from health_agent import config logger = logging.getLogger(__name__) # Both Swarm stacks running on the host: microservices + infra ("iklimco") # and the monitoring stack itself ("monitoring", this agent's own stack). STACK_NAMESPACES = ("iklimco", "monitoring") def check_stack_services(): start_time = time.time() try: client = docker.from_env() services = [] for namespace in STACK_NAMESPACES: services.extend(client.services.list(filters={"label": f"com.docker.stack.namespace={namespace}"})) # BE-* microservices are covered individually by the actuator checks # (checks/actuator.py, driven by monitors.yml's microservice_monitors), # which already detect missing/short replicas via DNS + health status. # Excluded here so a single service incident doesn't also flip this # aggregate down and fire a second, redundant notification. microservice_names = {m['service'] for m in config.load_microservice_monitors()} # Gather swarm services by short name, skipping microservice-covered ones swarm_services_by_name = {} for svc in services: name = svc.name short_name = name for namespace in STACK_NAMESPACES: prefix = f"{namespace}_" if name.startswith(prefix): short_name = name[len(prefix):] break if short_name in microservice_names: continue swarm_services_by_name[short_name] = svc missing_or_down = [] total_services = 0 for short_name, svc in swarm_services_by_name.items(): spec = svc.attrs.get('Spec', {}) mode = spec.get('Mode', {}) expected = 0 if 'Replicated' in mode: expected = mode['Replicated'].get('Replicas', 0) elif 'Global' in mode: nodes = client.nodes.list() expected = sum(1 for n in nodes if n.attrs.get('Status', {}).get('State') == 'ready') if expected == 0: continue # Intentionally scaled down and no config expectation total_services += 1 tasks = client.api.tasks(filters={"service": svc.id, "desired-state": "running"}) running_tasks = sum(1 for t in tasks if t.get('Status', {}).get('State') == 'running') if running_tasks < expected: missing_or_down.append(f"{short_name} {running_tasks}/{expected}") ping_ms = int((time.time() - start_time) * 1000) if not missing_or_down: msg = f"{total_services} services, all replicas running" push("Stack Services", "up", msg, ping_ms) else: msg = ", ".join(missing_or_down) push("Stack Services", "down", msg, ping_ms) except Exception as e: ping_ms = int((time.time() - start_time) * 1000) logger.error(f"Stack services check failed: {e}") push("Stack Services", "down", str(e), ping_ms)