Systemd Services & Monitoring

#!/bin/bash
# /opt/scripts/health-check-instances.sh

# Get all instances from database
instances=$(python3 -c "
from backend.db.engine import sessionLocal
from backend.db.models import WhiteLabelInstance
db = sessionLocal()
instances = db.query(WhiteLabelInstance).all()
for i in instances:
    print(f'{i.instance_id}:{i.port}')
")

for entry in $instances; do
    IFS=':' read -r uuid port <<< "$entry"
    
    # HTTP health check
    if curl -sf "http://localhost:$port/api/health" > /dev/null; then
        echo "✓ Instance $uuid (port $port) is healthy"
    else
        echo "✗ Instance $uuid (port $port) is DOWN"
        
        # Send alert (email, Slack, etc.)
        # mail -s "Alert: Instance $uuid DOWN" admin@example.com
        
        # Auto restart (optional)
        # systemctl restart ov-panel-instance@$uuid
    fi
done

#!/bin/bash
# /opt/scripts/check-instance-errors.sh

for logfile in /opt/ov-panel-instances/instance-*/logs/error.log; do
    uuid=$(echo $logfile | grep -oP 'instance-\K[^/]+')
    
    # Check for errors in last 1 hour
    errors=$(find $logfile -mmin -60 -exec grep -i "error\|exception\|critical" {} \; | wc -l)
    
    if [ $errors -gt 10 ]; then
        echo "WARNING: Instance $uuid has $errors errors in last hour"
        
        # Send alert
        # Example: Slack webhook
        # curl -X POST -H 'Content-type: application/json' \
        #   --data "{\"text\":\"Instance $uuid has $errors errors\"}" \
        #   https://hooks.slack.com/services/YOUR/WEBHOOK/URL
    fi
done

#!/bin/bash
# /opt/scripts/backup-instances.sh

BACKUP_DIR="/opt/backups/instances"
DATE=$(date +%Y%m%d_%H%M%S)

mkdir -p $BACKUP_DIR

# Backup mỗi instance database
for instance in /opt/ov-panel-instances/instance-*/; do
    uuid=$(basename $instance | sed 's/instance-//')
    db_file="$instance/data/ov-panel.db"
    
    if [ -f "$db_file" ]; then
        cp "$db_file" "$BACKUP_DIR/${uuid}_${DATE}.db"
        echo "Backed up instance $uuid"
    fi
done

# Cleanup backups older than 30 days
find $BACKUP_DIR -name "*.db" -mtime +30 -delete

echo "Backup completed at $(date)"