Murat ÖZDEMİR
03a3c5fb70
Adds two-layer BE-* microservice monitoring: an aggregate Stack Services replica check across the iklimco and monitoring Swarm stacks (checks/swarm_services.py), and per-service actuator/DNS health checks driven by a new microservice_monitors section in monitors.yml (checks/actuator.py). BE-* services are excluded from the Stack Services aggregate so a single service incident produces exactly one Slack alert instead of two. Fixes found during QA of the initial implementation: RabbitMQ host derivation generated nonexistent numbered DNS names instead of using the real single-service topology, RabbitMQ and Patroni cluster checks did not push down status on node/member shortfall, and the etcd cluster check still used a hardcoded node list and quorum threshold instead of deriving them from CLUSTER_SIZE_ETCD. Also hardens Docker event crash alerting: excludes ephemeral Gitea Actions runner containers from crash notifications, skips alerts for containers Swarm intentionally stopped (rolling update/scale-down, detected via task DesiredState) instead of alerting on every deploy, and adds human-readable exit code descriptions to the Slack message. Uptime Kuma monitor names are now prefixed with "iklim [env]" since test and prod share one Kuma instance and unprefixed names collided. setup_uptime_kuma.py gained an ensure_push_monitor helper and a microservice push-monitor creation pass. Removed state.py and the unused restart_threshold config field (dead code, no longer referenced anywhere). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
80 lines
2.7 KiB
Python
80 lines
2.7 KiB
Python
import socket
|
|
import time
|
|
import logging
|
|
import requests
|
|
from health_agent.uptime_kuma import push
|
|
from health_agent.checks.http import http_check
|
|
from health_agent import config
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
def tcp_check(host, port, timeout=3):
|
|
start_time = time.time()
|
|
try:
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
sock.settimeout(timeout)
|
|
result = sock.connect_ex((host, port))
|
|
sock.close()
|
|
ping_ms = int((time.time() - start_time) * 1000)
|
|
if result == 0:
|
|
return True, ping_ms, None
|
|
else:
|
|
return False, ping_ms, f"Port {port} is closed or unreachable"
|
|
except Exception as e:
|
|
ping_ms = int((time.time() - start_time) * 1000)
|
|
return False, ping_ms, str(e)
|
|
|
|
def check_etcd_cluster():
|
|
nodes = [host for host, _ in config.etcd_hosts]
|
|
start_t = time.time()
|
|
|
|
healthy_count = 0
|
|
leader = None
|
|
errors = []
|
|
|
|
for node in nodes:
|
|
# 1. TCP Check on 2379
|
|
tcp_ok, ms, tcp_err = tcp_check(node, 2379)
|
|
if not tcp_ok:
|
|
errors.append(f"{node} port 2379 unreachable")
|
|
continue
|
|
|
|
# 2. HTTP Health check
|
|
url = f"http://{node}:2379/health"
|
|
http_ok, resp, ms, http_err = http_check(url, timeout=3)
|
|
|
|
if http_ok and resp:
|
|
data = resp.json()
|
|
if data.get("health") == "true":
|
|
healthy_count += 1
|
|
else:
|
|
errors.append(f"{node} unhealthy")
|
|
else:
|
|
errors.append(f"{node} health endpoint unreachable")
|
|
|
|
# 3. Leader check from /v3/maintenance/status
|
|
if not leader and tcp_ok:
|
|
status_url = f"http://{node}:2379/v3/maintenance/status"
|
|
try:
|
|
r = requests.post(status_url, json={}, timeout=3)
|
|
if r.status_code == 200:
|
|
status_data = r.json()
|
|
leader_id = status_data.get("leader")
|
|
header_member_id = status_data.get("header", {}).get("member_id")
|
|
if leader_id and leader_id == header_member_id:
|
|
leader = node
|
|
except Exception:
|
|
pass
|
|
|
|
ping_ms = int((time.time() - start_t) * 1000)
|
|
quorum_threshold = (len(nodes) // 2) + 1
|
|
|
|
if healthy_count == len(nodes):
|
|
leader_info = f" | leader: {leader}" if leader else ""
|
|
msg = f"{healthy_count}/{len(nodes)} healthy{leader_info}"
|
|
push("Etcd Cluster", "up", msg, ping_ms)
|
|
else:
|
|
quorum_msg = f" | quorum at risk ({healthy_count}/{len(nodes)})" if healthy_count < quorum_threshold else ""
|
|
msg = " | ".join(errors) + quorum_msg
|
|
push("Etcd Cluster", "down", msg, ping_ms)
|