Murat ÖZDEMİR
03a3c5fb70
Adds two-layer BE-* microservice monitoring: an aggregate Stack Services replica check across the iklimco and monitoring Swarm stacks (checks/swarm_services.py), and per-service actuator/DNS health checks driven by a new microservice_monitors section in monitors.yml (checks/actuator.py). BE-* services are excluded from the Stack Services aggregate so a single service incident produces exactly one Slack alert instead of two. Fixes found during QA of the initial implementation: RabbitMQ host derivation generated nonexistent numbered DNS names instead of using the real single-service topology, RabbitMQ and Patroni cluster checks did not push down status on node/member shortfall, and the etcd cluster check still used a hardcoded node list and quorum threshold instead of deriving them from CLUSTER_SIZE_ETCD. Also hardens Docker event crash alerting: excludes ephemeral Gitea Actions runner containers from crash notifications, skips alerts for containers Swarm intentionally stopped (rolling update/scale-down, detected via task DesiredState) instead of alerting on every deploy, and adds human-readable exit code descriptions to the Slack message. Uptime Kuma monitor names are now prefixed with "iklim [env]" since test and prod share one Kuma instance and unprefixed names collided. setup_uptime_kuma.py gained an ensure_push_monitor helper and a microservice push-monitor creation pass. Removed state.py and the unused restart_threshold config field (dead code, no longer referenced anywhere). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
94 lines
3.2 KiB
Python
94 lines
3.2 KiB
Python
import time
|
|
import socket
|
|
import logging
|
|
import requests
|
|
import concurrent.futures
|
|
from health_agent.uptime_kuma import push
|
|
from health_agent import config
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
def check_single_actuator(ip, port, path, timeout):
|
|
try:
|
|
url = f"http://{ip}:{port}{path}"
|
|
resp = requests.get(url, timeout=timeout)
|
|
if resp.status_code != 200:
|
|
return False, f"HTTP {resp.status_code}"
|
|
|
|
try:
|
|
data = resp.json()
|
|
if data.get("status") == "UP":
|
|
return True, ""
|
|
return False, f"status: {data.get('status', 'unknown')}"
|
|
except ValueError:
|
|
return True, "" # JSON parse failed, but HTTP 200 is sufficient
|
|
except Exception as e:
|
|
return False, str(e)
|
|
|
|
def check_service(svc):
|
|
start_time = time.time()
|
|
name = svc["name"]
|
|
service = svc["service"]
|
|
port = svc["port"]
|
|
path = svc.get("path", "/actuator/health")
|
|
timeout = svc.get("timeout", 5)
|
|
expected_replicas = svc.get("expected_replicas")
|
|
|
|
try:
|
|
# Resolve A records
|
|
try:
|
|
addr_info = socket.getaddrinfo(service, port, socket.AF_INET, socket.SOCK_STREAM)
|
|
ips = list(set(item[4][0] for item in addr_info))
|
|
except socket.gaierror as e:
|
|
ping_ms = int((time.time() - start_time) * 1000)
|
|
push(name, "down", f"DNS resolution failed: {e}", ping_ms)
|
|
return
|
|
|
|
if not ips:
|
|
ping_ms = int((time.time() - start_time) * 1000)
|
|
push(name, "down", "DNS resolution failed: no IP returned", ping_ms)
|
|
return
|
|
|
|
healthy_count = 0
|
|
unhealthy_ips = []
|
|
|
|
# Check all IPs
|
|
for ip in ips:
|
|
is_up, err = check_single_actuator(ip, port, path, timeout)
|
|
if is_up:
|
|
healthy_count += 1
|
|
else:
|
|
unhealthy_ips.append(f"{ip} {err}")
|
|
|
|
ping_ms = int((time.time() - start_time) * 1000)
|
|
|
|
if unhealthy_ips:
|
|
total = len(ips)
|
|
unhealthy_count = len(unhealthy_ips)
|
|
msg = f"{unhealthy_count}/{total} unhealthy: {', '.join(unhealthy_ips)}"
|
|
push(name, "down", msg, ping_ms)
|
|
return
|
|
|
|
if expected_replicas is not None and healthy_count < expected_replicas:
|
|
resolved = len(ips)
|
|
msg = f"{healthy_count}/{expected_replicas} replicas healthy (expected {expected_replicas}, resolved {resolved})"
|
|
push(name, "down", msg, ping_ms)
|
|
return
|
|
|
|
total = expected_replicas if expected_replicas is not None else len(ips)
|
|
push(name, "up", f"{healthy_count}/{total} replicas healthy", ping_ms)
|
|
|
|
except Exception as e:
|
|
ping_ms = int((time.time() - start_time) * 1000)
|
|
logger.error(f"Actuator check failed for {name}: {e}")
|
|
push(name, "down", str(e), ping_ms)
|
|
|
|
def run_all_actuator_checks():
|
|
monitors = config.load_microservice_monitors()
|
|
if not monitors:
|
|
return
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
|
|
futures = [executor.submit(check_service, m) for m in monitors]
|
|
concurrent.futures.wait(futures)
|