Monitoring

# Show status of all containers including health
docker compose ps

docker inspect --format='{{.State.Health.Status}}' <container-name>

# UnrealIRCd — test TLS connection
openssl s_client -connect localhost:6697 -servername irc.localhost </dev/null 2>/dev/null | head -5

# UnrealIRCd — JSON-RPC info endpoint (requires auth; use WEBPANEL_RPC_USER/PASSWORD from .env)
curl -s -u "$WEBPANEL_RPC_USER:$WEBPANEL_RPC_PASSWORD" \
  -X POST http://localhost:8600/ \
  -H "Content-Type: application/json" \
  -d '{"jsonrpc":"2.0","method":"rpc.info","params":{},"id":1}' | jq .

# Atheme — check the HTTP API responds
curl -sf http://localhost:8081/ && echo "Atheme HTTP OK"

# Prosody — HTTP status page
curl -sf http://localhost:5280/status

# Prosody — XMPP C2S port connectivity
nc -zv localhost 5222

# Bridge — check process is running
docker exec atl-bridge pgrep -f "bridge.__main__" && echo "Bridge running"

# The Lounge — HTTP connectivity
curl -sf http://localhost:9000/ -o /dev/null && echo "The Lounge OK"

# WebPanel — HTTP connectivity
curl -sf http://localhost:8080/ -o /dev/null && echo "WebPanel OK"

# Follow all service logs
docker compose logs --follow

# Follow a specific service
docker compose logs --follow atl-irc-server

# Show last 100 lines for a service
docker compose logs --tail=100 atl-bridge

# Search logs for errors
docker compose logs atl-bridge 2>&1 | grep -i error

# Filter by time range (last hour)
docker compose logs --since=1h atl-xmpp-server

# UnrealIRCd — connection errors and netsplits
docker compose logs atl-irc-server 2>&1 | grep -iE "error|split|refused"

# Atheme — failed authentication attempts
docker compose logs atl-irc-server 2>&1 | grep -i "SASL"

# Prosody — authentication failures and certificate issues
docker compose logs atl-xmpp-server 2>&1 | grep -iE "auth.*fail|certificate|tls"

# Bridge — Discord or IRC connection issues
docker compose logs atl-bridge 2>&1 | grep -iE "disconnect|error|reconnect"

# Check the IRC TLS certificate expiry date
openssl x509 -enddate -noout -in data/certs/live/irc.localhost/fullchain.pem

# Check all certificates in data/certs/
find data/certs/live -name "fullchain.pem" -exec \
  sh -c 'echo "{}:"; openssl x509 -enddate -noout -in "{}"' \;

# Disk usage of data directories
du -sh data/*/

# Container memory and CPU usage
docker stats --no-stream --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"

scrape_configs:
  # Prosody XMPP server metrics (OpenMetrics)
  - job_name: xmpp-prosody
    metrics_path: /metrics
    scheme: http
    scrape_interval: 30s
    scrape_timeout: 10s
    static_configs:
      - targets: ["atl-xmpp-server:5280"] # Use container name within Docker network
        labels:
          service: prosody
          environment: production

# Example Prometheus alerting rules — add to your rules file
groups:
  - name: atl-chat
    rules:
      # Prosody down
      - alert: ProsodyDown
        expr: up{job="xmpp-prosody"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Prosody XMPP server is down"

      # High XMPP auth failure rate
      - alert: ProsodyHighAuthFailures
        expr: rate(prosody_c2s_auth_failure_total[5m]) > 10
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "High XMPP authentication failure rate"

      # High Prosody memory usage
      - alert: ProsodyHighMemory
        expr: prosody_memory_usage_bytes > 500000000
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Prosody memory usage exceeds 500 MB"

#!/usr/bin/env bash
# /etc/cron.d/atl-health-check — runs every 5 minutes
# Checks all atl.chat services and sends a notification on failure

SERVICES=("atl-irc-server" "atl-irc-services" "atl-xmpp-server" "atl-bridge")

for svc in "${SERVICES[@]}"; do
  status=$(docker inspect --format='{{.State.Health.Status}}' "$svc" 2>/dev/null)
  if [ "$status" != "healthy" ]; then
    echo "ALERT: $svc is $status" | mail -s "atl.chat health alert" ops@example.com
  fi
done