/google/appengine/tools/devappserver2/health_check_service.py

https://github.com/theosp/google_appengine · Python · 169 lines · 82 code · 20 blank · 67 comment · 12 complexity · 8a32fce6affca0e26c1f8872018a98e6 MD5 · raw file

  1. #!/usr/bin/env python
  2. #
  3. # Copyright 2007 Google Inc.
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License");
  6. # you may not use this file except in compliance with the License.
  7. # You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. #
  17. """A health checking implementation for the SDK.
  18. <scrub>
  19. This code attempts to match the production implementation as closely as
  20. possible in apphosting/runtime/vm/vm_health_check.cc.
  21. One instance of HealthChecker should be created per instance.Instance that has
  22. health checking enabled. The HealthChecker instance needs to be started, but
  23. will stop itself automatically.
  24. </scrub>
  25. """
  26. import logging
  27. import threading
  28. import time
  29. from google.appengine.api import request_info
  30. from google.appengine.tools.devappserver2 import start_response_utils
  31. class _HealthCheckState(object):
  32. """A class to track the state of a health checked instance."""
  33. def __init__(self):
  34. """Initializes a _HealthCheckState object."""
  35. self.consecutive_healthy_responses = 0
  36. self.consecutive_unhealthy_responses = 0
  37. self.is_last_successful = False
  38. def update(self, healthy):
  39. """Updates the state.
  40. Args:
  41. healthy: Bool indicating whether the last attempt was healthy.
  42. """
  43. self.is_last_successful = healthy
  44. if healthy:
  45. self.consecutive_healthy_responses += 1
  46. self.consecutive_unhealthy_responses = 0
  47. else:
  48. self.consecutive_healthy_responses = 0
  49. self.consecutive_unhealthy_responses += 1
  50. def __str__(self):
  51. """Outputs the state in a readable way for logging."""
  52. tmpl = '{number} consecutive {state} responses.'
  53. if self.consecutive_healthy_responses:
  54. number = self.consecutive_healthy_responses
  55. state = 'HEALTHY'
  56. else:
  57. number = self.consecutive_unhealthy_responses
  58. state = 'UNHEALTHY'
  59. return tmpl.format(number=number, state=state)
  60. class HealthChecker(object):
  61. """A class to perform health checks for an instance.
  62. This class uses the settings specified in appinfo.VmHealthCheck and the
  63. callback specified to check the health of the specified instance. When
  64. appropriate, this class changes the state of the specified instance so it is
  65. placed into or taken out of load balancing. This class will also use another
  66. callback to restart the instance, if necessary.
  67. """
  68. def __init__(self, instance, config, send_request, restart):
  69. """Initializes a HealthChecker object.
  70. Args:
  71. instance: An instance.Instance object.
  72. config: An appinfo.VmHealthCheck object.
  73. send_request: A function to call that makes the health check request.
  74. restart: A function to call that restarts the instance.
  75. """
  76. self._instance = instance
  77. self._config = config
  78. self._send_request = send_request
  79. self._restart = restart
  80. def start(self):
  81. """Starts the health checks."""
  82. logging.info('Health checks starting for instance %s.',
  83. self._instance.instance_id)
  84. loop = threading.Thread(target=self._loop)
  85. loop.daemon = True
  86. loop.start()
  87. def _should_continue(self):
  88. return self._running and not self._instance.has_quit
  89. def _loop(self):
  90. """Performs health checks and updates state over time."""
  91. state = _HealthCheckState()
  92. self._instance.set_health(False)
  93. self._running = True
  94. while self._should_continue():
  95. logging.debug('Performing health check for instance %s.',
  96. self._instance.instance_id)
  97. self._do_health_check(state)
  98. logging.debug('Health check state for instance: %s: %s',
  99. self._instance.instance_id, state)
  100. time.sleep(self._config.check_interval_sec)
  101. def _do_health_check(self, state):
  102. health = self._get_health_check_response(state.is_last_successful)
  103. state.update(health)
  104. self._maybe_update_instance(state)
  105. def _maybe_update_instance(self, state):
  106. """Performs any required actions on the instance based on the state.
  107. Args:
  108. state: A _HealthCheckState object.
  109. """
  110. if (state.consecutive_unhealthy_responses >=
  111. self._config.unhealthy_threshold):
  112. self._instance.set_health(False)
  113. elif (state.consecutive_healthy_responses >=
  114. self._config.healthy_threshold):
  115. self._instance.set_health(True)
  116. if (state.consecutive_unhealthy_responses >=
  117. self._config.restart_threshold):
  118. self._restart_instance()
  119. def _get_health_check_response(self, is_last_successful):
  120. """Sends the health check request and checks the result.
  121. Args:
  122. is_last_successful: Whether the last request was successful.
  123. Returns:
  124. A bool indicating whether or not the instance is healthy.
  125. """
  126. start_response = start_response_utils.CapturingStartResponse()
  127. try:
  128. response = self._send_request(start_response, is_last_successful)
  129. except request_info.Error, e:
  130. logging.warning('Health check failure for instance %s. Exception: %s',
  131. self._instance.instance_id, e)
  132. return False
  133. logging.debug('Health check response %s and status %s for instance %s.',
  134. response, start_response.status, self._instance.instance_id)
  135. return response == ['ok'] and start_response.status == '200 OK'
  136. def _restart_instance(self):
  137. """Restarts the running instance, and stops the current health checker."""
  138. logging.warning('Restarting instance %s because of failed health checks.',
  139. self._instance.instance_id)
  140. self._running = False
  141. self._restart()