diff --git a/.gitignore b/.gitignore index 7f960e3..b1c482c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ health-agent/.venv/* health-agent/prod-env/ health-agent/test-env/ health-agent/.env* + +.claude/ diff --git a/health-agent/README.md b/health-agent/README.md index 1789d0a..d1d9bed 100644 --- a/health-agent/README.md +++ b/health-agent/README.md @@ -36,9 +36,10 @@ Environment_Monitoring/health-agent/ │ ├── config.py # .env + uk_tokens.yml yükler; ortam ayarlarını expose eder │ ├── uptime_kuma.py # push(token, status, msg, ping_ms) yardımcısı │ ├── slack.py # notify(webhook, source, priority, title, detail, uk_group_url) — kaynak etiketli + UK grup linki -│ ├── state.py # restart sayısı gibi session-arası state'i dosyaya yazar/okur │ ├── checks/ -│ │ ├── swarm.py # Docker API: node listesi, servis replica sayıları +│ │ ├── swarm.py # Docker API: node listesi (sadece Swarm node kontrolü) +│ │ ├── swarm_services.py # Docker API: iklimco stack servislerinin replica kontrolü +│ │ ├── actuator.py # Mikroservisler için HTTP actuator health (IP/replica bazlı) check │ │ ├── http.py # genel HTTP check + uygulama bazlı parser'lar (Patroni, Vault, RabbitMQ...) │ │ ├── tcp.py # TCP port erişilebilirliği │ │ ├── tls.py # TLS sertifika son kullanma tarihi (dosyadan veya handshake'den) @@ -61,6 +62,8 @@ Environment_Monitoring/health-agent/ **`config/monitors.yml`** — Tüm monitor, group, tag ve status page tanımları bu dosyadadır. Yeni bir monitor eklemek için kod değişikliği gerekmez; `monitors.yml`'e yeni bir blok eklenir ve `setup_uptime_kuma.py` çalıştırılır. +**Mikroservis Ekleme:** `microservice_monitors` bölümü, BE servislerinin Uptime Kuma'ya (actuator ve swarm replica bazında) eklenmesini otomatikleştirir. Yeni bir BE servisi eklendiğinde kod değişikliği gerekmez; sadece bu bölüme servis tanımı ve opsiyonel `replicas.` beklentisi yazılıp setup script çalıştırılır. `replicas.`, ortam bazlı beklenen replica sayısıdır; Docker "desired" değerinden bağımsız çalışır (yanlış scale-down veya deploy edilmeme durumlarını alarm olarak yakalar). Servisin replica değerini kalıcı olarak değiştiriyorsanız, `monitors.yml` dosyasını da güncelleyip imajı rebuild etmelisiniz (çünkü bu dosya imaja gömülüdür). + **`.env`** — Runtime değişkenler: | Variable | Description | @@ -70,14 +73,16 @@ Environment_Monitoring/health-agent/ | `CLUSTER_SIZE_ETCD` | etcd node count (prod: 3, test: 1) | | `CLUSTER_SIZE_PATRONI` | Patroni node count | | `CLUSTER_SIZE_MONGODB` | MongoDB node count | -| `CLUSTER_SIZE_RABBITMQ` | RabbitMQ node count | +| `CLUSTER_SIZE_RABBITMQ` | Expected RabbitMQ cluster member count (RabbitMQ runs as a single Swarm service — this only sets the expected total, not per-node hostnames) | | `CLUSTER_SIZE_VAULT` | Vault node count | | `REDIS_MODE` | `sentinel` or `standalone` | | `EXTERNAL_DOMAIN` | Base domain — `iklim.co` in both environments | | `EXTERNAL_SUBDOMAIN_SUFFIX` | Subdomain suffix — empty for prod, `-test` for test → `api-test.iklim.co` | | `SLACK_WEBHOOK_IKLIM_{ENV}_OPS` | Direct Slack webhook for container crash/OOM events — e.g. `SLACK_WEBHOOK_IKLIM_PROD_OPS` | +| `ETCD_HOSTS` | etcd node list (comma-separated `host:port`) — e.g. `etcd-01:2379,etcd-02:2379`; falls back to `etcd-01..0N` derived from `CLUSTER_SIZE_ETCD` | | `PATRONI_HOSTS` | Patroni node list (comma-separated `host:port`) — e.g. `patroni-01:8008,patroni-02:8008` | | `VAULT_HOSTS` | Vault node subdomain list (comma-separated) — e.g. `vault-1,vault-2,vault-3` | +| `RABBITMQ_HOSTS` | Optional RabbitMQ endpoint override (comma-separated `host:port`); defaults to the single `rabbitmq` Swarm service — normally leave unset | | `RABBITMQ_USER` / `RABBITMQ_PASS` | RabbitMQ management credentials | | `MONGO_URI` | MongoDB connection URI | | `REDIS_PASSWORD` | Redis / Sentinel password | @@ -86,6 +91,10 @@ Environment_Monitoring/health-agent/ | `STORAGEBOX_PATH` | StorageBox mount path for filesystem check | | `APISIX_ADMIN_KEY` | APISIX admin API key for health check | +**Altyapı Beklentileri (CLUSTER_SIZE_*)**: Altyapı check'leri, `*_HOSTS` listesi `.env` içinde tanımlıysa onu kullanır, tanımsız/boş ise `CLUSTER_SIZE_*` sayısına göre (ör. `etcd-01`, `etcd-02`...) beklenen host listesini otomatik türetir. Bu sayede cluster node sayısını artırıp azaltmak kod veya imaj rebuild gerektirmez. + +**Monitör Adlandırma Konvansiyonu**: Test ve prod aynı Uptime Kuma'yı paylaştığı için monitörler UK arayüzünde `iklim [] ` formatıyla oluşturulur. Ancak `uk_tokens.yml` dosyası ve kod içi `push()` fonksiyonu, prefix'siz ham ad (ör. "Stack Services") ile çalışmaya devam eder. + Check periyotları `monitors.yml`'de her monitor için tanımlanır; `.env`'e eklenmez. Push token'ları `config/generated/uk_tokens.yml`'den otomatik okunur — bu dosya `setup_uptime_kuma.py` tarafından üretilir ve `.env`'e elle kopyalanmaz. diff --git a/health-agent/config/monitors.yml b/health-agent/config/monitors.yml index 576d4ee..a8a4e73 100644 --- a/health-agent/config/monitors.yml +++ b/health-agent/config/monitors.yml @@ -76,77 +76,72 @@ groups: notifications: [slack-low] tags: [internal, observability] children: [Prometheus, Grafana, Portainer, Loki, Ext Https Portainer, Ext Https Apigw] + - name: "Microservices" + status_page: "iklim-{env}-ops" + notifications: [slack-high] + tags: [internal, high] + children: [Stack Services, Svc Auth, Svc Account, Svc Lightning, Svc Thunderstorm, Svc Precipitation, Svc Nowcast Point Alarm, Svc Nowcast Geo Alarm, Svc Forecast, Svc Forecast Point Alarm, Svc Enroute] push_monitors: - name: Swarm Cluster interval: 75 heartbeat_retries: 1 tags: [internal, infrastructure, high] - restart_threshold: 1 - name: Vault Cluster interval: 75 heartbeat_retries: 1 tags: [internal, infrastructure, high] - restart_threshold: 1 - name: Etcd Cluster interval: 75 heartbeat_retries: 1 tags: [internal, database, high] - restart_threshold: 1 - name: Patroni Cluster interval: 75 heartbeat_retries: 1 tags: [internal, database, high] - restart_threshold: 1 - name: Mongodb Replicaset interval: 120 heartbeat_retries: 1 tags: [internal, database, high] - restart_threshold: 1 - name: Apisix Gateway interval: 75 heartbeat_retries: 1 tags: [internal, gateway, high] - restart_threshold: 1 - name: Rabbitmq Cluster interval: 75 heartbeat_retries: 1 tags: [internal, gateway, medium] - restart_threshold: 3 - name: Redis Sentinel interval: 75 heartbeat_retries: 1 tags: [internal, database, medium] - restart_threshold: 3 - name: Swag Tls interval: 3600 heartbeat_retries: 1 tags: [internal, infrastructure, medium] - restart_threshold: 3 - name: Storagebox Mount interval: 300 heartbeat_retries: 1 tags: [internal, infrastructure, medium] - restart_threshold: 1 - name: Prometheus interval: 120 heartbeat_retries: 1 tags: [internal, observability, low] - restart_threshold: 5 - name: Grafana interval: 120 heartbeat_retries: 1 tags: [internal, observability, low] - restart_threshold: 5 - name: Portainer interval: 120 heartbeat_retries: 1 tags: [internal, observability, low] - restart_threshold: 5 - name: Loki interval: 120 heartbeat_retries: 1 tags: [internal, observability, low] - restart_threshold: 5 + - name: Stack Services + interval: 120 + heartbeat_retries: 1 + tags: [internal, high] http_monitors: - name: Ext Https Api url: "https://api{suffix}.{domain}/health" @@ -190,7 +185,76 @@ status_pages: - "Gateway & Messaging" - "External Availability - Critical" - "External Availability - General" + - "Microservices" - slug: "iklim-{env}-tools" title: "iklim.co [{env}] Tools" public: false groups: ["Observability"] + +microservice_monitors: + defaults: + interval: 120 + heartbeat_retries: 1 + path: /actuator/health + timeout: 5 + services: + - name: Svc Auth + service: auth-service + port: 8081 + replicas: + prod: 3 + test: 1 + - name: Svc Account + service: account-service + port: 8082 + replicas: + prod: 3 + test: 1 + - name: Svc Lightning + service: lightning-service + port: 8085 + replicas: + prod: 3 + test: 1 + - name: Svc Thunderstorm + service: thunderstorm-service + port: 8086 + replicas: + prod: 3 + test: 1 + - name: Svc Precipitation + service: precipitation-service + port: 8087 + replicas: + prod: 3 + test: 1 + - name: Svc Nowcast Point Alarm + service: nowcast-point-alarm-service + port: 8088 + replicas: + prod: 3 + test: 1 + - name: Svc Nowcast Geo Alarm + service: nowcast-geo-alarm-service + port: 8089 + replicas: + prod: 3 + test: 1 + - name: Svc Forecast + service: forecast-service + port: 8090 + replicas: + prod: 3 + test: 1 + - name: Svc Forecast Point Alarm + service: forecast-point-alarm-service + port: 8091 + replicas: + prod: 3 + test: 1 + - name: Svc Enroute + service: enroute-service + port: 8092 + replicas: + prod: 3 + test: 1 diff --git a/health-agent/scripts/setup_uptime_kuma.py b/health-agent/scripts/setup_uptime_kuma.py index d34830c..1c497bd 100644 --- a/health-agent/scripts/setup_uptime_kuma.py +++ b/health-agent/scripts/setup_uptime_kuma.py @@ -40,6 +40,42 @@ def find_group_notifications(monitor_name, groups, notification_map): return None +def ensure_push_monitor(api, m_name, formatted_name, interval, parent_group_id, notif_ids, existing_monitors, tokens, new_monitor_ids, dry_run): + logger.info(f"Processing push monitor: {formatted_name}") + if not dry_run: + if formatted_name in existing_monitors: + logger.info(f"Monitor {formatted_name} already exists. Updating...") + m_id = existing_monitors[formatted_name]['id'] + tokens[m_name] = existing_monitors[formatted_name]['pushToken'] + + kwargs = { + "interval": interval + } + if parent_group_id: + kwargs["parent"] = parent_group_id + if notif_ids: + kwargs["notificationIDList"] = notif_ids + + try: + api.edit_monitor(m_id, **kwargs) + except Exception as e: + logger.warning(f"Failed to edit push monitor {formatted_name}: {e}") + else: + logger.info(f"Creating push monitor: {formatted_name}") + kwargs = { + "type": MonitorType.PUSH, + "name": formatted_name, + "interval": interval, + "parent": parent_group_id + } + if notif_ids: + kwargs["notificationIDList"] = notif_ids + result = api.add_monitor(**kwargs) + new_monitor_ids[m_name] = result['monitorID'] + else: + tokens[m_name] = "dummy_token_dry_run" + + def setup_uptime_kuma(dry_run=False, only=None): env_name = os.getenv("ENV", "test") @@ -138,43 +174,32 @@ def setup_uptime_kuma(dry_run=False, only=None): if only and m_name != only: continue + formatted_name = f"{project} [{env_name}] {m_name}" m_interval = pm.get("interval", 60) parent_group_id = find_parent_group(m_name, config.get("groups", []), group_map) notif_ids = find_group_notifications(m_name, config.get("groups", []), notification_map) - logger.info(f"Processing push monitor: {m_name}") - if not dry_run: - if m_name in existing_monitors: - logger.info(f"Monitor {m_name} already exists. Updating...") - m_id = existing_monitors[m_name]['id'] - tokens[m_name] = existing_monitors[m_name]['pushToken'] + ensure_push_monitor(api, m_name, formatted_name, m_interval, parent_group_id, notif_ids, existing_monitors, tokens, new_monitor_ids, dry_run) - kwargs = { - "interval": m_interval - } - if parent_group_id: - kwargs["parent"] = parent_group_id - if notif_ids: - kwargs["notificationIDList"] = notif_ids - - try: - api.edit_monitor(m_id, **kwargs) - except Exception as e: - logger.warning(f"Failed to edit push monitor {m_name}: {e}") - else: - logger.info(f"Creating push monitor: {m_name}") - kwargs = { - "type": MonitorType.PUSH, - "name": m_name, - "interval": m_interval, - "parent": parent_group_id - } - if notif_ids: - kwargs["notificationIDList"] = notif_ids - result = api.add_monitor(**kwargs) - new_monitor_ids[m_name] = result['monitorID'] - else: - tokens[m_name] = "dummy_token_dry_run" + # 2b. Microservice push monitors + ms_config = config.get("microservice_monitors", {}) + if ms_config: + defaults = ms_config.get("defaults", {}) + for svc in ms_config.get("services", []): + m_name = svc["name"] + if only and m_name != only: + continue + + envs = svc.get("envs") + if envs and env_name not in envs: + continue + + formatted_name = f"{project} [{env_name}] {m_name}" + m_interval = svc.get("interval", defaults.get("interval", 120)) + parent_group_id = find_parent_group(m_name, config.get("groups", []), group_map) + notif_ids = find_group_notifications(m_name, config.get("groups", []), notification_map) + + ensure_push_monitor(api, m_name, formatted_name, m_interval, parent_group_id, notif_ids, existing_monitors, tokens, new_monitor_ids, dry_run) # Fetch push tokens for newly created monitors in one batch call. # Calling api.get_monitors() per-monitor races with WebSocket event delivery; @@ -201,14 +226,15 @@ def setup_uptime_kuma(dry_run=False, only=None): parent_group_id = find_parent_group(m_name, config.get("groups", []), group_map) notif_ids = find_group_notifications(m_name, config.get("groups", []), notification_map) - logger.info(f"Processing HTTP monitor: {m_name} -> {url}") + formatted_name = f"{project} [{env_name}] {m_name}" + logger.info(f"Processing HTTP monitor: {formatted_name} -> {url}") if not dry_run: - if m_name in existing_monitors: - logger.info(f"Monitor {m_name} already exists. Updating...") - m_id = existing_monitors[m_name]['id'] + if formatted_name in existing_monitors: + logger.info(f"Monitor {formatted_name} already exists. Updating...") + m_id = existing_monitors[formatted_name]['id'] kwargs = { "type": MonitorType.HTTP, - "name": m_name, + "name": formatted_name, "url": url, "interval": interval, "accepted_statuscodes": accepted_statuscodes, @@ -220,12 +246,12 @@ def setup_uptime_kuma(dry_run=False, only=None): try: api.edit_monitor(m_id, **kwargs) except Exception as e: - logger.warning(f"Failed to edit HTTP monitor {m_name}: {e}") + logger.warning(f"Failed to edit HTTP monitor {formatted_name}: {e}") else: try: kwargs = { "type": MonitorType.HTTP, - "name": m_name, + "name": formatted_name, "url": url, "interval": interval, "accepted_statuscodes": accepted_statuscodes, @@ -235,9 +261,9 @@ def setup_uptime_kuma(dry_run=False, only=None): if notif_ids: kwargs["notificationIDList"] = notif_ids api.add_monitor(**kwargs) - logger.info(f"Created HTTP monitor: {m_name}") + logger.info(f"Created HTTP monitor: {formatted_name}") except Exception as e: - logger.warning(f"Failed to create HTTP monitor {m_name}: {e}") + logger.warning(f"Failed to create HTTP monitor {formatted_name}: {e}") # 4. DNS Monitors for dm in config.get("dns_monitors", []): @@ -250,14 +276,15 @@ def setup_uptime_kuma(dry_run=False, only=None): parent_group_id = find_parent_group(m_name, config.get("groups", []), group_map) notif_ids = find_group_notifications(m_name, config.get("groups", []), notification_map) - logger.info(f"Processing DNS monitor: {m_name} -> {hostname}") + formatted_name = f"{project} [{env_name}] {m_name}" + logger.info(f"Processing DNS monitor: {formatted_name} -> {hostname}") if not dry_run: - if m_name in existing_monitors: - logger.info(f"Monitor {m_name} already exists. Updating...") - m_id = existing_monitors[m_name]['id'] + if formatted_name in existing_monitors: + logger.info(f"Monitor {formatted_name} already exists. Updating...") + m_id = existing_monitors[formatted_name]['id'] kwargs = { "type": MonitorType.DNS, - "name": m_name, + "name": formatted_name, "hostname": hostname, "port": 53, "accepted_statuscodes": ["200-299"], @@ -274,12 +301,12 @@ def setup_uptime_kuma(dry_run=False, only=None): try: api.edit_monitor(m_id, **kwargs) except Exception as e: - logger.warning(f"Failed to edit DNS monitor {m_name}: {e}") + logger.warning(f"Failed to edit DNS monitor {formatted_name}: {e}") else: try: kwargs = { "type": MonitorType.DNS, - "name": m_name, + "name": formatted_name, "hostname": hostname, "port": 53, "accepted_statuscodes": ["200-299"], @@ -294,9 +321,9 @@ def setup_uptime_kuma(dry_run=False, only=None): if notif_ids: kwargs["notificationIDList"] = notif_ids api.add_monitor(**kwargs) - logger.info(f"Created DNS monitor: {m_name}") + logger.info(f"Created DNS monitor: {formatted_name}") except Exception as e: - logger.warning(f"Failed to create DNS monitor {m_name}: {e}") + logger.warning(f"Failed to create DNS monitor {formatted_name}: {e}") # 5. Ping Monitors (generated from nodes config) ping_cfg = config.get("ping_monitors", {}) @@ -312,14 +339,15 @@ def setup_uptime_kuma(dry_run=False, only=None): parent_group_id = find_parent_group(m_name, config.get("groups", []), group_map) notif_ids = find_group_notifications(m_name, config.get("groups", []), notification_map) - logger.info(f"Processing Ping monitor: {m_name} -> {ip}") + formatted_name = f"{project} [{env_name}] {m_name}" + logger.info(f"Processing Ping monitor: {formatted_name} -> {ip}") if not dry_run: - if m_name in existing_monitors: - logger.info(f"Monitor {m_name} already exists. Updating...") - m_id = existing_monitors[m_name]['id'] + if formatted_name in existing_monitors: + logger.info(f"Monitor {formatted_name} already exists. Updating...") + m_id = existing_monitors[formatted_name]['id'] kwargs = { "type": MonitorType.PING, - "name": m_name, + "name": formatted_name, "hostname": ip, "interval": ping_interval, "maxretries": ping_retries, @@ -331,12 +359,12 @@ def setup_uptime_kuma(dry_run=False, only=None): try: api.edit_monitor(m_id, **kwargs) except Exception as e: - logger.warning(f"Failed to edit Ping monitor {m_name}: {e}") + logger.warning(f"Failed to edit Ping monitor {formatted_name}: {e}") else: try: kwargs = { "type": MonitorType.PING, - "name": m_name, + "name": formatted_name, "hostname": ip, "interval": ping_interval, "maxretries": ping_retries, @@ -346,9 +374,9 @@ def setup_uptime_kuma(dry_run=False, only=None): if notif_ids: kwargs["notificationIDList"] = notif_ids api.add_monitor(**kwargs) - logger.info(f"Created Ping monitor: {m_name}") + logger.info(f"Created Ping monitor: {formatted_name}") except Exception as e: - logger.warning(f"Failed to create Ping monitor {m_name}: {e}") + logger.warning(f"Failed to create Ping monitor {formatted_name}: {e}") for i, node in enumerate(env_nodes.get("db", []), 1): m_name = f"Ext Ping Db{i:02d}" @@ -358,14 +386,15 @@ def setup_uptime_kuma(dry_run=False, only=None): parent_group_id = find_parent_group(m_name, config.get("groups", []), group_map) notif_ids = find_group_notifications(m_name, config.get("groups", []), notification_map) - logger.info(f"Processing Ping monitor: {m_name} -> {ip}") + formatted_name = f"{project} [{env_name}] {m_name}" + logger.info(f"Processing Ping monitor: {formatted_name} -> {ip}") if not dry_run: - if m_name in existing_monitors: - logger.info(f"Monitor {m_name} already exists. Updating...") - m_id = existing_monitors[m_name]['id'] + if formatted_name in existing_monitors: + logger.info(f"Monitor {formatted_name} already exists. Updating...") + m_id = existing_monitors[formatted_name]['id'] kwargs = { "type": MonitorType.PING, - "name": m_name, + "name": formatted_name, "hostname": ip, "interval": ping_interval, "maxretries": ping_retries, @@ -377,12 +406,12 @@ def setup_uptime_kuma(dry_run=False, only=None): try: api.edit_monitor(m_id, **kwargs) except Exception as e: - logger.warning(f"Failed to edit Ping monitor {m_name}: {e}") + logger.warning(f"Failed to edit Ping monitor {formatted_name}: {e}") else: try: kwargs = { "type": MonitorType.PING, - "name": m_name, + "name": formatted_name, "hostname": ip, "interval": ping_interval, "maxretries": ping_retries, @@ -392,9 +421,9 @@ def setup_uptime_kuma(dry_run=False, only=None): if notif_ids: kwargs["notificationIDList"] = notif_ids api.add_monitor(**kwargs) - logger.info(f"Created Ping monitor: {m_name}") + logger.info(f"Created Ping monitor: {formatted_name}") except Exception as e: - logger.warning(f"Failed to create Ping monitor {m_name}: {e}") + logger.warning(f"Failed to create Ping monitor {formatted_name}: {e}") # 6. Status Pages if api: diff --git a/health-agent/src/health_agent/checks/actuator.py b/health-agent/src/health_agent/checks/actuator.py new file mode 100644 index 0000000..1e9d1ff --- /dev/null +++ b/health-agent/src/health_agent/checks/actuator.py @@ -0,0 +1,93 @@ +import time +import socket +import logging +import requests +import concurrent.futures +from health_agent.uptime_kuma import push +from health_agent import config + +logger = logging.getLogger(__name__) + +def check_single_actuator(ip, port, path, timeout): + try: + url = f"http://{ip}:{port}{path}" + resp = requests.get(url, timeout=timeout) + if resp.status_code != 200: + return False, f"HTTP {resp.status_code}" + + try: + data = resp.json() + if data.get("status") == "UP": + return True, "" + return False, f"status: {data.get('status', 'unknown')}" + except ValueError: + return True, "" # JSON parse failed, but HTTP 200 is sufficient + except Exception as e: + return False, str(e) + +def check_service(svc): + start_time = time.time() + name = svc["name"] + service = svc["service"] + port = svc["port"] + path = svc.get("path", "/actuator/health") + timeout = svc.get("timeout", 5) + expected_replicas = svc.get("expected_replicas") + + try: + # Resolve A records + try: + addr_info = socket.getaddrinfo(service, port, socket.AF_INET, socket.SOCK_STREAM) + ips = list(set(item[4][0] for item in addr_info)) + except socket.gaierror as e: + ping_ms = int((time.time() - start_time) * 1000) + push(name, "down", f"DNS resolution failed: {e}", ping_ms) + return + + if not ips: + ping_ms = int((time.time() - start_time) * 1000) + push(name, "down", "DNS resolution failed: no IP returned", ping_ms) + return + + healthy_count = 0 + unhealthy_ips = [] + + # Check all IPs + for ip in ips: + is_up, err = check_single_actuator(ip, port, path, timeout) + if is_up: + healthy_count += 1 + else: + unhealthy_ips.append(f"{ip} {err}") + + ping_ms = int((time.time() - start_time) * 1000) + + if unhealthy_ips: + total = len(ips) + unhealthy_count = len(unhealthy_ips) + msg = f"{unhealthy_count}/{total} unhealthy: {', '.join(unhealthy_ips)}" + push(name, "down", msg, ping_ms) + return + + if expected_replicas is not None and healthy_count < expected_replicas: + resolved = len(ips) + msg = f"{healthy_count}/{expected_replicas} replicas healthy (expected {expected_replicas}, resolved {resolved})" + push(name, "down", msg, ping_ms) + return + + total = expected_replicas if expected_replicas is not None else len(ips) + push(name, "up", f"{healthy_count}/{total} replicas healthy", ping_ms) + + except Exception as e: + ping_ms = int((time.time() - start_time) * 1000) + logger.error(f"Actuator check failed for {name}: {e}") + push(name, "down", str(e), ping_ms) + +def run_all_actuator_checks(): + monitors = config.load_microservice_monitors() + if not monitors: + return + + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + futures = [executor.submit(check_service, m) for m in monitors] + concurrent.futures.wait(futures) diff --git a/health-agent/src/health_agent/checks/http.py b/health-agent/src/health_agent/checks/http.py index 257d721..23e42a6 100644 --- a/health-agent/src/health_agent/checks/http.py +++ b/health-agent/src/health_agent/checks/http.py @@ -4,6 +4,7 @@ import logging import requests from requests.auth import HTTPBasicAuth from health_agent.uptime_kuma import push +from health_agent import config import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -29,15 +30,7 @@ def http_check(url, expected_status=None, auth=None, verify_ssl=True, timeout=5, return False, None, ping_ms, str(e) def check_patroni_cluster(): - hosts_env = os.getenv("PATRONI_HOSTS", "patroni-01:8008,patroni-02:8008,patroni-03:8008") - nodes = [] - for h in hosts_env.split(","): - h = h.strip() - if ":" in h: - host, port = h.rsplit(":", 1) - nodes.append((host, int(port))) - else: - nodes.append((h, 8008)) + nodes = config.patroni_hosts cluster_data = None error_msg = "All Patroni nodes unreachable" start_t = time.time() @@ -74,44 +67,64 @@ def check_patroni_cluster(): down_nodes = [f"{r[0]} state: {r[2]}" for r in replicas if r[2] not in ("running", "streaming")] msg = f"no leader detected | " + " ".join(down_nodes) push("Patroni Cluster", "down", msg, ping_ms) - else: - lag_strs = [] - for name, lag, state in replicas: - lag_mb = lag / (1024*1024) if isinstance(lag, (int, float)) else 0 - lag_strs.append(f"{name} (lag:{lag_mb:.0f}MB)") - - msg = f"leader: {leader} | replicas: " + " ".join(lag_strs) - push("Patroni Cluster", "up", msg, ping_ms) + return + + expected = config.CLUSTER_SIZE_PATRONI + if len(members) < expected: + lag_strs = [f"{name} (lag:{(lag / (1024*1024)) if isinstance(lag, (int, float)) else 0:.0f}MB)" for name, lag, state in replicas] + msg = f"leader: {leader} | only {len(members)}/{expected} members visible | replicas: " + " ".join(lag_strs) + push("Patroni Cluster", "down", msg, ping_ms) + return + + lag_strs = [] + for name, lag, state in replicas: + lag_mb = lag / (1024*1024) if isinstance(lag, (int, float)) else 0 + lag_strs.append(f"{name} (lag:{lag_mb:.0f}MB)") + + msg = f"leader: {leader} | replicas: " + " ".join(lag_strs) + push("Patroni Cluster", "up", msg, ping_ms) def check_rabbitmq_cluster(): - url = "http://rabbitmq:15672/api/healthchecks/node" user = os.getenv("RABBITMQ_USER", "guest") password = os.getenv("RABBITMQ_PASS", "guest") auth = HTTPBasicAuth(user, password) - ok, resp, ping_ms, err = http_check(url, auth=auth) + nodes = config.rabbitmq_hosts + ok2, resp2 = False, None + error_msg = "All RabbitMQ nodes unreachable" + start_t = time.time() - if ok: - ok2, resp2, _, _ = http_check("http://rabbitmq:15672/api/nodes", auth=auth) - nodes_running = 0 - total_nodes = 3 - - if ok2 and resp2: - data = resp2.json() - nodes_running = len([n for n in data if n.get("running")]) - total_nodes = len(data) + for host, port in nodes: + url = f"http://{host}:{port}/api/nodes" + ok, resp, _, err = http_check(url, auth=auth, timeout=3) + if ok and resp: + ok2 = True + resp2 = resp + break + elif err: + error_msg = f"{host}:{port} error: {err}" - alarms = [n.get("name") for n in data if n.get("mem_alarm") or n.get("disk_free_alarm")] - if alarms: - msg = f"disk/mem alarm active on {','.join(alarms)}" - push("Rabbitmq Cluster", "down", msg, ping_ms) - return - - msg = f"{nodes_running}/{total_nodes} nodes running" - push("Rabbitmq Cluster", "up", msg, ping_ms) - else: - msg = err or f"HTTP {resp.status_code if resp else 'Unknown'}" + ping_ms = int((time.time() - start_t) * 1000) + + if not ok2 or not resp2: + push("Rabbitmq Cluster", "down", error_msg, ping_ms) + return + + data = resp2.json() + nodes_running = len([n for n in data if n.get("running")]) + total_nodes = config.CLUSTER_SIZE_RABBITMQ + + alarms = [n.get("name") for n in data if n.get("mem_alarm") or n.get("disk_free_alarm")] + if alarms: + msg = f"disk/mem alarm active on {','.join(alarms)}" push("Rabbitmq Cluster", "down", msg, ping_ms) + return + + msg = f"{nodes_running}/{total_nodes} nodes running" + if nodes_running < total_nodes: + push("Rabbitmq Cluster", "down", msg, ping_ms) + else: + push("Rabbitmq Cluster", "up", msg, ping_ms) def check_apisix(): url = "http://apisix:9180/apisix/admin/routes" @@ -125,8 +138,7 @@ def check_apisix(): push("Apisix Gateway", "down", f"admin API unreachable: {err or resp.status_code}", ping_ms) def check_vault(): - hosts_env = os.getenv("VAULT_HOSTS", "vault") - nodes = [h.strip() for h in hosts_env.split(",")] + nodes = [host for host, _ in config.vault_hosts] domain = os.getenv("EXTERNAL_DOMAIN", "iklim.co") unsealed_count = 0 total = len(nodes) diff --git a/health-agent/src/health_agent/checks/swarm_services.py b/health-agent/src/health_agent/checks/swarm_services.py new file mode 100644 index 0000000..6f37ba3 --- /dev/null +++ b/health-agent/src/health_agent/checks/swarm_services.py @@ -0,0 +1,78 @@ +import time +import docker +import logging +from health_agent.uptime_kuma import push +from health_agent import config + +logger = logging.getLogger(__name__) + +# Both Swarm stacks running on the host: microservices + infra ("iklimco") +# and the monitoring stack itself ("monitoring", this agent's own stack). +STACK_NAMESPACES = ("iklimco", "monitoring") + +def check_stack_services(): + start_time = time.time() + try: + client = docker.from_env() + services = [] + for namespace in STACK_NAMESPACES: + services.extend(client.services.list(filters={"label": f"com.docker.stack.namespace={namespace}"})) + + # BE-* microservices are covered individually by the actuator checks + # (checks/actuator.py, driven by monitors.yml's microservice_monitors), + # which already detect missing/short replicas via DNS + health status. + # Excluded here so a single service incident doesn't also flip this + # aggregate down and fire a second, redundant notification. + microservice_names = {m['service'] for m in config.load_microservice_monitors()} + + # Gather swarm services by short name, skipping microservice-covered ones + swarm_services_by_name = {} + for svc in services: + name = svc.name + short_name = name + for namespace in STACK_NAMESPACES: + prefix = f"{namespace}_" + if name.startswith(prefix): + short_name = name[len(prefix):] + break + if short_name in microservice_names: + continue + swarm_services_by_name[short_name] = svc + + missing_or_down = [] + total_services = 0 + + for short_name, svc in swarm_services_by_name.items(): + spec = svc.attrs.get('Spec', {}) + mode = spec.get('Mode', {}) + expected = 0 + + if 'Replicated' in mode: + expected = mode['Replicated'].get('Replicas', 0) + elif 'Global' in mode: + nodes = client.nodes.list() + expected = sum(1 for n in nodes if n.attrs.get('Status', {}).get('State') == 'ready') + + if expected == 0: + continue # Intentionally scaled down and no config expectation + + total_services += 1 + tasks = client.api.tasks(filters={"service": svc.id, "desired-state": "running"}) + running_tasks = sum(1 for t in tasks if t.get('Status', {}).get('State') == 'running') + + if running_tasks < expected: + missing_or_down.append(f"{short_name} {running_tasks}/{expected}") + + ping_ms = int((time.time() - start_time) * 1000) + + if not missing_or_down: + msg = f"{total_services} services, all replicas running" + push("Stack Services", "up", msg, ping_ms) + else: + msg = ", ".join(missing_or_down) + push("Stack Services", "down", msg, ping_ms) + + except Exception as e: + ping_ms = int((time.time() - start_time) * 1000) + logger.error(f"Stack services check failed: {e}") + push("Stack Services", "down", str(e), ping_ms) diff --git a/health-agent/src/health_agent/checks/tcp.py b/health-agent/src/health_agent/checks/tcp.py index 5b00816..c712a99 100644 --- a/health-agent/src/health_agent/checks/tcp.py +++ b/health-agent/src/health_agent/checks/tcp.py @@ -4,6 +4,7 @@ import logging import requests from health_agent.uptime_kuma import push from health_agent.checks.http import http_check +from health_agent import config logger = logging.getLogger(__name__) @@ -24,24 +25,24 @@ def tcp_check(host, port, timeout=3): return False, ping_ms, str(e) def check_etcd_cluster(): - nodes = ["etcd-01", "etcd-02", "etcd-03"] + nodes = [host for host, _ in config.etcd_hosts] start_t = time.time() - + healthy_count = 0 leader = None errors = [] - + for node in nodes: # 1. TCP Check on 2379 tcp_ok, ms, tcp_err = tcp_check(node, 2379) if not tcp_ok: errors.append(f"{node} port 2379 unreachable") continue - + # 2. HTTP Health check url = f"http://{node}:2379/health" http_ok, resp, ms, http_err = http_check(url, timeout=3) - + if http_ok and resp: data = resp.json() if data.get("health") == "true": @@ -50,7 +51,7 @@ def check_etcd_cluster(): errors.append(f"{node} unhealthy") else: errors.append(f"{node} health endpoint unreachable") - + # 3. Leader check from /v3/maintenance/status if not leader and tcp_ok: status_url = f"http://{node}:2379/v3/maintenance/status" @@ -66,12 +67,13 @@ def check_etcd_cluster(): pass ping_ms = int((time.time() - start_t) * 1000) - + quorum_threshold = (len(nodes) // 2) + 1 + if healthy_count == len(nodes): leader_info = f" | leader: {leader}" if leader else "" msg = f"{healthy_count}/{len(nodes)} healthy{leader_info}" push("Etcd Cluster", "up", msg, ping_ms) else: - quorum_msg = f" | quorum at risk ({healthy_count}/{len(nodes)})" if healthy_count < 3 else "" + quorum_msg = f" | quorum at risk ({healthy_count}/{len(nodes)})" if healthy_count < quorum_threshold else "" msg = " | ".join(errors) + quorum_msg push("Etcd Cluster", "down", msg, ping_ms) diff --git a/health-agent/src/health_agent/config.py b/health-agent/src/health_agent/config.py index c00ad28..f716790 100644 --- a/health-agent/src/health_agent/config.py +++ b/health-agent/src/health_agent/config.py @@ -15,6 +15,34 @@ REDIS_MODE = os.getenv("REDIS_MODE", "sentinel") EXTERNAL_DOMAIN = os.getenv("EXTERNAL_DOMAIN", "iklim.co") EXTERNAL_SUBDOMAIN_SUFFIX = os.getenv("EXTERNAL_SUBDOMAIN_SUFFIX", "") +def get_cluster_hosts(env_var_hosts, size, prefix, port, start_index=1, default_hosts=None): + env_hosts = os.getenv(env_var_hosts) + if env_hosts: + nodes = [] + for h in env_hosts.split(","): + h = h.strip() + if ":" in h: + host, p = h.rsplit(":", 1) + nodes.append((host, int(p))) + else: + nodes.append((h, port)) + return nodes + + if default_hosts is not None: + return default_hosts + + return [(f"{prefix}-{i:02d}", port) for i in range(start_index, start_index + size)] + +etcd_hosts = get_cluster_hosts("ETCD_HOSTS", CLUSTER_SIZE_ETCD, "etcd", 2379) +patroni_hosts = get_cluster_hosts("PATRONI_HOSTS", CLUSTER_SIZE_PATRONI, "patroni", 8008) +vault_hosts = get_cluster_hosts("VAULT_HOSTS", CLUSTER_SIZE_VAULT, "vault", 8200) +# RabbitMQ runs as a single Swarm service (not per-node DNS names like etcd/patroni); +# the management API on that one service already reports the whole cluster's node list. +rabbitmq_hosts = get_cluster_hosts( + "RABBITMQ_HOSTS", CLUSTER_SIZE_RABBITMQ, "rabbitmq", 15672, + default_hosts=[("rabbitmq", 15672)], +) + def load_uk_tokens(): try: with open("config/generated/uk_tokens.yml", "r") as f: @@ -23,3 +51,36 @@ def load_uk_tokens(): return {} UK_TOKENS = load_uk_tokens() + +def load_microservice_monitors(): + try: + with open("config/monitors.yml", "r") as f: + data = yaml.safe_load(f) or {} + except (FileNotFoundError, OSError): + return [] + + ms_config = data.get("microservice_monitors", {}) + if not ms_config: + return [] + + defaults = ms_config.get("defaults", {}) + services = ms_config.get("services", []) + + result = [] + for svc in services: + envs = svc.get("envs") + if envs and ENV not in envs: + continue + + merged = defaults.copy() + merged.update(svc) + + replicas_map = svc.get("replicas", {}) + if isinstance(replicas_map, dict): + merged["expected_replicas"] = replicas_map.get(ENV) + else: + merged["expected_replicas"] = None + + result.append(merged) + + return result diff --git a/health-agent/src/health_agent/events/docker_events.py b/health-agent/src/health_agent/events/docker_events.py index 36a149c..c61505b 100644 --- a/health-agent/src/health_agent/events/docker_events.py +++ b/health-agent/src/health_agent/events/docker_events.py @@ -1,4 +1,5 @@ import os +import signal import docker import threading import logging @@ -7,26 +8,75 @@ from health_agent.slack import notify logger = logging.getLogger(__name__) -def parse_and_notify(event): +# Ephemeral Gitea Actions runner job containers (act_runner) — torn down after +# every workflow run, often with a non-zero/SIGKILL exit code that has nothing +# to do with actual service health. Not part of the iklimco/monitoring stacks. +EXCLUDED_CONTAINER_PREFIXES = ("GITEA-ACTIONS-TASK",) + +# Human-readable meaning for common container exit codes, so Slack alerts +# don't require looking up a signal/exit-code table by hand. +EXIT_CODE_MEANINGS = { + "1": "Application error (uncaught exception or generic failure)", + "2": "Misconfiguration or shell builtin misuse", + "126": "Container command found but not executable", + "127": "Container command not found", + "130": "Interrupted (SIGINT)", + "137": "Killed (SIGKILL) — OOM killer or forced stop", + "139": "Segmentation fault (SIGSEGV)", + "143": "Terminated (SIGTERM) — graceful stop requested", +} + +def describe_exit_code(exit_code): + if exit_code in EXIT_CODE_MEANINGS: + return EXIT_CODE_MEANINGS[exit_code] + try: + code = int(exit_code) + if code > 128: + return f"Killed by signal {signal.Signals(code - 128).name}" + except ValueError: + pass + return "Unknown" + +def is_orchestrated_stop(client, attrs, container_name): + """True if Swarm itself asked for this task to stop (rolling update, + scale-down, service removal) rather than the container dying on its own. + """ + task_id = attrs.get('com.docker.swarm.task.id') + if not task_id and '.' in container_name: + task_id = container_name.rsplit('.', 1)[-1] + if not task_id: + return False + try: + task = client.api.inspect_task(task_id) + return task.get('DesiredState') == 'shutdown' + except Exception: + return False + +def parse_and_notify(event, client): attrs = event.get('Actor', {}).get('Attributes', {}) container_name = attrs.get('name', 'unknown') + + if container_name.startswith(EXCLUDED_CONTAINER_PREFIXES): + return + exit_code = attrs.get('exitCode', '0') - + if exit_code == '0': return - + + if is_orchestrated_stop(client, attrs, container_name): + return + is_oom = (exit_code == '137') - + env = os.getenv("ENV", "test").upper() webhook_env_name = f"SLACK_WEBHOOK_IKLIM_{env}_OPS" - + priority = "High" if is_oom else "Medium" title = f"[Health Agent / Events] Container Crashed ({container_name})" - - detail = f"Container: {container_name}\nExit Code: {exit_code}" - if is_oom: - detail += "\nReason: OOM Killed (Out Of Memory) or SIGKILL" - + + detail = f"Container: {container_name}\nExit Code: {exit_code} ({describe_exit_code(exit_code)})" + notify( webhook_env=webhook_env_name, source="health-agent-events", @@ -43,7 +93,7 @@ def event_listener_loop(): filters = {"type": "container", "event": "die"} for event in client.events(decode=True, filters=filters): try: - parse_and_notify(event) + parse_and_notify(event, client) except Exception as e: logger.error(f"Error parsing event: {e}", exc_info=True) except Exception as e: diff --git a/health-agent/src/health_agent/main.py b/health-agent/src/health_agent/main.py index 5321a1d..175b19b 100644 --- a/health-agent/src/health_agent/main.py +++ b/health-agent/src/health_agent/main.py @@ -4,6 +4,8 @@ import logging import json import threading from health_agent.checks import swarm +from health_agent.checks.swarm_services import check_stack_services +from health_agent.checks.actuator import run_all_actuator_checks from health_agent.checks.http import run_all_http_checks from health_agent.checks.tcp import check_etcd_cluster from health_agent.checks.tls import check_swag_tls @@ -40,6 +42,16 @@ def run_checks(): except Exception as e: logger.error(f"Error checking Swarm cluster: {e}") + try: + check_stack_services() + except Exception as e: + logger.error(f"Error checking stack services: {e}") + + try: + run_all_actuator_checks() + except Exception as e: + logger.error(f"Error checking actuator endpoints: {e}") + try: run_all_http_checks() except Exception as e: diff --git a/health-agent/src/health_agent/state.py b/health-agent/src/health_agent/state.py deleted file mode 100644 index 753c49a..0000000 --- a/health-agent/src/health_agent/state.py +++ /dev/null @@ -1,19 +0,0 @@ -import json -import os -from pathlib import Path - -STATE_FILE = Path("config/generated/state.json") - -def load_state(): - if not STATE_FILE.exists(): - return {} - try: - with open(STATE_FILE, "r") as f: - return json.load(f) - except Exception: - return {} - -def save_state(state): - STATE_FILE.parent.mkdir(parents=True, exist_ok=True) - with open(STATE_FILE, "w") as f: - json.dump(state, f)