import logging import requests from celery import shared_task from django.utils import timezone from labs.models import LabSSHServer logger = logging.getLogger(__name__) @shared_task def ssh_servers_health_check(): ssh_servers = LabSSHServer.objects.all() for ssh_server in ssh_servers: try: response = requests.get( f"https://{ssh_server.domain}/health", timeout=5, ) if response.status_code == 200: # increment success streak; require 2 consecutive successes to recover if ssh_server.status == LabSSHServer.Status.UNHEALTHY: ssh_server.healthy_success_streak = ( ssh_server.healthy_success_streak or 0 ) + 1 if ssh_server.healthy_success_streak >= 2: ssh_server.status = LabSSHServer.Status.HEALTHY ssh_server.healthy_success_streak = 0 ssh_server.save( update_fields=["status", "healthy_success_streak"] ) else: ssh_server.save(update_fields=["healthy_success_streak"]) else: if ssh_server.healthy_success_streak != 0: ssh_server.healthy_success_streak = 0 ssh_server.save(update_fields=["healthy_success_streak"]) else: raise Exception( f"SSH server is unhealthy, ssh_server_id: {ssh_server.id}, " f"status code: {response.status_code}, response: {response.text[:200]}" ) except Exception as e: logger.error( f"Unexpected error during health check, ssh_server_id: {ssh_server.id}, error: {e}" ) update_fields = ["status", "healthy_success_streak"] if ssh_server.status != LabSSHServer.Status.UNHEALTHY: ssh_server.last_unhealthy_at = timezone.now() update_fields.append("last_unhealthy_at") ssh_server.status = LabSSHServer.Status.UNHEALTHY ssh_server.healthy_success_streak = 0 ssh_server.save(update_fields=update_fields)