Murat ÖZDEMİR
03a3c5fb70
Adds two-layer BE-* microservice monitoring: an aggregate Stack Services replica check across the iklimco and monitoring Swarm stacks (checks/swarm_services.py), and per-service actuator/DNS health checks driven by a new microservice_monitors section in monitors.yml (checks/actuator.py). BE-* services are excluded from the Stack Services aggregate so a single service incident produces exactly one Slack alert instead of two. Fixes found during QA of the initial implementation: RabbitMQ host derivation generated nonexistent numbered DNS names instead of using the real single-service topology, RabbitMQ and Patroni cluster checks did not push down status on node/member shortfall, and the etcd cluster check still used a hardcoded node list and quorum threshold instead of deriving them from CLUSTER_SIZE_ETCD. Also hardens Docker event crash alerting: excludes ephemeral Gitea Actions runner containers from crash notifications, skips alerts for containers Swarm intentionally stopped (rolling update/scale-down, detected via task DesiredState) instead of alerting on every deploy, and adds human-readable exit code descriptions to the Slack message. Uptime Kuma monitor names are now prefixed with "iklim [env]" since test and prod share one Kuma instance and unprefixed names collided. setup_uptime_kuma.py gained an ensure_push_monitor helper and a microservice push-monitor creation pass. Removed state.py and the unused restart_threshold config field (dead code, no longer referenced anywhere). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
261 lines
6.7 KiB
YAML
261 lines
6.7 KiB
YAML
version: "1"
|
|
project: "iklim"
|
|
domain:
|
|
base: "iklim.co"
|
|
nodes:
|
|
prod:
|
|
service:
|
|
- name: iklim-app-01
|
|
ip: "178.104.210.41"
|
|
- name: iklim-app-02
|
|
ip: "178.105.69.1"
|
|
- name: iklim-app-03
|
|
ip: "178.104.219.3"
|
|
db:
|
|
- name: iklim-db-01
|
|
ip: "159.69.117.158"
|
|
- name: iklim-db-02
|
|
ip: "178.104.219.162"
|
|
- name: iklim-db-03
|
|
ip: "159.69.115.105"
|
|
test:
|
|
service:
|
|
- name: iklim-app-01
|
|
ip: "167.235.194.61"
|
|
db:
|
|
- name: iklim-db-01
|
|
ip: "167.235.205.93"
|
|
tags:
|
|
- external
|
|
- internal
|
|
- high
|
|
- medium
|
|
- low
|
|
- database
|
|
- gateway
|
|
- infrastructure
|
|
- observability
|
|
notifications:
|
|
slack-high:
|
|
type: slack
|
|
webhook_env: UK_SLACK_WEBHOOK_HIGH
|
|
slack-medium:
|
|
type: slack
|
|
webhook_env: UK_SLACK_WEBHOOK_MEDIUM
|
|
slack-low:
|
|
type: slack
|
|
webhook_env: UK_SLACK_WEBHOOK_LOW
|
|
groups:
|
|
- name: "Infrastructure"
|
|
status_page: "iklim-{env}-ops"
|
|
notifications: [slack-high]
|
|
tags: [internal, infrastructure]
|
|
children: [Swarm Cluster, Vault Cluster, Storagebox Mount, Swag Tls]
|
|
- name: "Data Layer"
|
|
status_page: "iklim-{env}-ops"
|
|
notifications: [slack-high]
|
|
tags: [internal, database]
|
|
children: [Etcd Cluster, Patroni Cluster, Mongodb Replicaset]
|
|
- name: "Gateway & Messaging"
|
|
status_page: "iklim-{env}-ops"
|
|
notifications: [slack-high]
|
|
tags: [internal, gateway]
|
|
children: [Apisix Gateway, Rabbitmq Cluster, Redis Sentinel]
|
|
- name: "External Availability - Critical"
|
|
status_page: "iklim-{env}-ops"
|
|
notifications: [slack-high]
|
|
tags: [external, high]
|
|
children: [Ext Https Api, Ext Dns Api, Ext Dns Root, Ext Ping App01, Ext Ping App02, Ext Ping App03]
|
|
- name: "External Availability - General"
|
|
status_page: "iklim-{env}-ops"
|
|
notifications: [slack-medium]
|
|
tags: [external, medium]
|
|
children: [Ext Https Grafana, Ext Ping Db01, Ext Ping Db02, Ext Ping Db03]
|
|
- name: "Observability"
|
|
status_page: "iklim-{env}-tools"
|
|
notifications: [slack-low]
|
|
tags: [internal, observability]
|
|
children: [Prometheus, Grafana, Portainer, Loki, Ext Https Portainer, Ext Https Apigw]
|
|
- name: "Microservices"
|
|
status_page: "iklim-{env}-ops"
|
|
notifications: [slack-high]
|
|
tags: [internal, high]
|
|
children: [Stack Services, Svc Auth, Svc Account, Svc Lightning, Svc Thunderstorm, Svc Precipitation, Svc Nowcast Point Alarm, Svc Nowcast Geo Alarm, Svc Forecast, Svc Forecast Point Alarm, Svc Enroute]
|
|
push_monitors:
|
|
- name: Swarm Cluster
|
|
interval: 75
|
|
heartbeat_retries: 1
|
|
tags: [internal, infrastructure, high]
|
|
- name: Vault Cluster
|
|
interval: 75
|
|
heartbeat_retries: 1
|
|
tags: [internal, infrastructure, high]
|
|
- name: Etcd Cluster
|
|
interval: 75
|
|
heartbeat_retries: 1
|
|
tags: [internal, database, high]
|
|
- name: Patroni Cluster
|
|
interval: 75
|
|
heartbeat_retries: 1
|
|
tags: [internal, database, high]
|
|
- name: Mongodb Replicaset
|
|
interval: 120
|
|
heartbeat_retries: 1
|
|
tags: [internal, database, high]
|
|
- name: Apisix Gateway
|
|
interval: 75
|
|
heartbeat_retries: 1
|
|
tags: [internal, gateway, high]
|
|
- name: Rabbitmq Cluster
|
|
interval: 75
|
|
heartbeat_retries: 1
|
|
tags: [internal, gateway, medium]
|
|
- name: Redis Sentinel
|
|
interval: 75
|
|
heartbeat_retries: 1
|
|
tags: [internal, database, medium]
|
|
- name: Swag Tls
|
|
interval: 3600
|
|
heartbeat_retries: 1
|
|
tags: [internal, infrastructure, medium]
|
|
- name: Storagebox Mount
|
|
interval: 300
|
|
heartbeat_retries: 1
|
|
tags: [internal, infrastructure, medium]
|
|
- name: Prometheus
|
|
interval: 120
|
|
heartbeat_retries: 1
|
|
tags: [internal, observability, low]
|
|
- name: Grafana
|
|
interval: 120
|
|
heartbeat_retries: 1
|
|
tags: [internal, observability, low]
|
|
- name: Portainer
|
|
interval: 120
|
|
heartbeat_retries: 1
|
|
tags: [internal, observability, low]
|
|
- name: Loki
|
|
interval: 120
|
|
heartbeat_retries: 1
|
|
tags: [internal, observability, low]
|
|
- name: Stack Services
|
|
interval: 120
|
|
heartbeat_retries: 1
|
|
tags: [internal, high]
|
|
http_monitors:
|
|
- name: Ext Https Api
|
|
url: "https://api{suffix}.{domain}/health"
|
|
accepted_statuscodes: ["200"]
|
|
interval: 60
|
|
- name: Ext Https Grafana
|
|
url: "https://grafana{suffix}.{domain}/api/health"
|
|
accepted_statuscodes: ["200"]
|
|
interval: 60
|
|
- name: Ext Https Portainer
|
|
url: "https://portainer{suffix}.{domain}"
|
|
accepted_statuscodes: ["200", "401", "403"]
|
|
interval: 120
|
|
- name: Ext Https Apigw
|
|
url: "https://apigw{suffix}.{domain}"
|
|
accepted_statuscodes: ["200", "401", "403"]
|
|
interval: 120
|
|
dns_monitors:
|
|
- name: Ext Dns Api
|
|
hostname: "api{suffix}.{domain}"
|
|
dns_resolve_type: A
|
|
interval: 60
|
|
- name: Ext Dns Root
|
|
hostname: "{domain}"
|
|
dns_resolve_type: A
|
|
interval: 60
|
|
ping_monitors:
|
|
interval: 60
|
|
max_retries: 1
|
|
status_pages:
|
|
- slug: "iklim-{env}-status"
|
|
title: "iklim.co API Status"
|
|
public: true
|
|
groups: ["External Availability - Critical"]
|
|
- slug: "iklim-{env}-ops"
|
|
title: "iklim.co [{env}] Infrastructure"
|
|
public: false
|
|
groups:
|
|
- "Infrastructure"
|
|
- "Data Layer"
|
|
- "Gateway & Messaging"
|
|
- "External Availability - Critical"
|
|
- "External Availability - General"
|
|
- "Microservices"
|
|
- slug: "iklim-{env}-tools"
|
|
title: "iklim.co [{env}] Tools"
|
|
public: false
|
|
groups: ["Observability"]
|
|
|
|
microservice_monitors:
|
|
defaults:
|
|
interval: 120
|
|
heartbeat_retries: 1
|
|
path: /actuator/health
|
|
timeout: 5
|
|
services:
|
|
- name: Svc Auth
|
|
service: auth-service
|
|
port: 8081
|
|
replicas:
|
|
prod: 3
|
|
test: 1
|
|
- name: Svc Account
|
|
service: account-service
|
|
port: 8082
|
|
replicas:
|
|
prod: 3
|
|
test: 1
|
|
- name: Svc Lightning
|
|
service: lightning-service
|
|
port: 8085
|
|
replicas:
|
|
prod: 3
|
|
test: 1
|
|
- name: Svc Thunderstorm
|
|
service: thunderstorm-service
|
|
port: 8086
|
|
replicas:
|
|
prod: 3
|
|
test: 1
|
|
- name: Svc Precipitation
|
|
service: precipitation-service
|
|
port: 8087
|
|
replicas:
|
|
prod: 3
|
|
test: 1
|
|
- name: Svc Nowcast Point Alarm
|
|
service: nowcast-point-alarm-service
|
|
port: 8088
|
|
replicas:
|
|
prod: 3
|
|
test: 1
|
|
- name: Svc Nowcast Geo Alarm
|
|
service: nowcast-geo-alarm-service
|
|
port: 8089
|
|
replicas:
|
|
prod: 3
|
|
test: 1
|
|
- name: Svc Forecast
|
|
service: forecast-service
|
|
port: 8090
|
|
replicas:
|
|
prod: 3
|
|
test: 1
|
|
- name: Svc Forecast Point Alarm
|
|
service: forecast-point-alarm-service
|
|
port: 8091
|
|
replicas:
|
|
prod: 3
|
|
test: 1
|
|
- name: Svc Enroute
|
|
service: enroute-service
|
|
port: 8092
|
|
replicas:
|
|
prod: 3
|
|
test: 1
|