/
opt
/
cloudlinux
/
venv
/
lib64
/
python3.11
/
site-packages
/
xray
/
agent
/
Upload Filee
HOME
# -*- coding: utf-8 -*- # Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2021 All Rights Reserved # # Licensed under CLOUD LINUX LICENSE AGREEMENT # http://cloudlinux.com/docs/LICENSE.TXT """ This module contains FaultDetector class, aimed to track throttling for incoming requests """ import logging import time from datetime import datetime, timedelta from threading import RLock from typing import Tuple, Any from xray.internal.constants import drop_after, check_period, throttling_threshold class FaultDetector: """ Fault Detector class """ def __init__(self): self.logger = logging.getLogger('fault_detector') self.throttling_stat = dict() self.lock = RLock() self.time_marker_file = '/usr/share/alt-php-xray/flush.latest' def __call__(self, t_key: Any, cpu_value: int) -> Tuple[bool, int]: """ Perform fault detection """ with self.lock: try: saved_cpu, _ = self.throttling_stat[t_key] except KeyError: hitting_limits, throttled_time = False, 0 else: hitting_limits, throttled_time = self._was_throttled_since( saved_cpu, cpu_value) self._flush_entry(t_key) return hitting_limits, throttled_time @property def timestamp(self) -> int: """ Current Unix timestamp """ return int(time.time()) @property def _drop_after(self) -> timedelta: """ Drop after value as datetime.timedelta """ return self.timedelta_in_minutes(drop_after) @property def _check_period(self) -> timedelta: """ Check period value as datetime.timedelta """ return self.timedelta_in_minutes(check_period) @property def should_flush(self) -> bool: """ If throttling_stat should be flushed right now regarding check period value """ try: with open(self.time_marker_file) as latest_info: latest_flush_time = int(latest_info.read().strip()) return self.current_delta(latest_flush_time) > self._check_period except (OSError, ValueError): with self.lock: self._update_latest_flush() return False @staticmethod def timedelta_in_minutes(mins: int) -> timedelta: """ Get timedelta object for given number of minutes """ return timedelta(minutes=mins) def current_delta(self, time_stamp: int) -> timedelta: """ Calculate timedelta for given time_stamp from current timestamp """ def _cast(ts: int) -> datetime: """ Make a datetime object from ts (int) """ return datetime.fromtimestamp(ts) return _cast(self.timestamp) - _cast(time_stamp) def save(self, key: Any, cpu: int) -> None: """ Save throttling entry for given key """ stamp = self.timestamp with self.lock: self.logger.debug('Adding new throttling entry %s: (%i, %i)', key, cpu, stamp) self.throttling_stat[key] = (cpu, stamp) self.logger.debug('Throttling entries total %i', len(self.throttling_stat)) def _flush_entry(self, key: Any) -> None: """ Remove throttling entry by given key """ with self.lock: self.logger.debug('Abandon throttling entry %s', key) self.throttling_stat.pop(key, 0) self.logger.debug('Throttling entries total %i', len(self.throttling_stat)) def _was_throttled_since(self, initial_value: int, current_value: int) -> Tuple[bool, int]: """ Check current throttling time of given lve_id vs given initial_value Return: tuple(fact of throttling, throttling time in ms) """ # initial values of throttling time are in nanoseconds! spent_time = (current_value - initial_value) // 1000000 # cast to ms throttling_fact = spent_time > throttling_threshold self.logger.debug('Throttling checked with %i: %s, %i', throttling_threshold, throttling_fact, spent_time) return throttling_fact, spent_time if spent_time > 0 else 0 def flush(self) -> None: """ Clear throttling stat dict """ if self.should_flush: with self.lock: self.logger.debug('Flush started, total entries %i', len(self.throttling_stat)) for key in self._expired_entries(): self._flush_entry(key) self._update_latest_flush() self.logger.debug('Flush finished, total entries %i', len(self.throttling_stat)) def _update_latest_flush(self) -> None: """ Write current timestamp into time marker file """ try: with open(self.time_marker_file, 'w') as latest_info: latest_info.write(str(self.timestamp)) except OSError: self.logger.error('Failed to save timestamp of latest flush') def _expired_entries(self) -> list: """ Collect expired throttling stat entries keys regarding the drop after value """ expired = list() for key, value in self.throttling_stat.items(): _, timestamp = value _delta = self.current_delta(timestamp) if _delta > self._drop_after: self.logger.debug('Expired entry detected %i: %s elapsed', key, _delta) expired.append(key) return expired