/branches/1.2.0/cron-ng/jobs/Scaling.php
PHP | 297 lines | 209 code | 45 blank | 43 comment | 36 complexity | fb5bba6dea8445db6cb06a0c2cf04efa MD5 | raw file
Possible License(s): LGPL-2.1, Apache-2.0, GPL-3.0
- <?php
- class Scalr_Cronjob_Scaling extends Scalr_System_Cronjob_MultiProcess_DefaultWorker
- {
- static function getConfig () {
- return array(
- "description" => "Scaling process",
- "processPool" => array(
- "daemonize" => true,
- "preventParalleling" => true,
- "size" => 5
- ),
- //"distributed" => true,
- //"iniFile" => dirname(dirname(__FILE__)) . "/distributed.ini",
- //"electorCls" => "Scalr_System_Cronjob_Distributed_DataCenterElector",
- //"leaderTimeout" => 120000, // 2min
- "fileName" => __FILE__,
- "memoryLimit" => 128000
- );
- }
-
- private $logger;
- private $db;
-
- public function __construct() {
- $this->logger = LoggerManager::getLogger(__CLASS__);
- $this->db = Core::GetDBInstance();
- }
-
- function startForking ($workQueue) {
- // Reopen DB connection after daemonizing
- $this->db = Core::GetDBInstance(null, true);
- }
-
- function startChild () {
- // Reopen DB connection in child
- $this->db = Core::GetDBInstance(null, true);
- // Reconfigure observers;
- Scalr::ReconfigureObservers();
- }
-
- function enqueueWork ($workQueue)
- {
- $this->logger->info("Fetching completed farms...");
-
- $rows = $this->db->GetAll("SELECT farms.*, clients.isactive FROM farms
- INNER JOIN clients ON clients.id = farms.clientid WHERE clients.isactive='1' AND farms.status=?",
- array(FARM_STATUS::RUNNING)
- );
- $this->logger->info("Found ".count($rows)." farms.");
-
- foreach ($rows as $row)
- {
- $workQueue->put($row["id"]);
- }
- }
-
- function handleWork ($farmId)
- {
- $farminfo = $this->db->GetRow("SELECT farms.*, clients.isactive FROM farms
- INNER JOIN clients ON clients.id = farms.clientid WHERE farms.id = ?", array($farmId));
-
- $GLOBALS["SUB_TRANSACTIONID"] = abs(crc32(posix_getpid().$farmId));
- $GLOBALS["LOGGER_FARMID"] = $farminfo["id"];
-
- $this->logger->info("[{$GLOBALS["SUB_TRANSACTIONID"]}] Begin polling farm (ID: {$farminfo['id']}, Name: {$farminfo['name']}, Status: {$farminfo['status']})");
-
- $farm_roles = $this->db->GetAll("SELECT ami_id, id, replace_to_ami FROM farm_roles WHERE farmid=? ORDER BY launch_index ASC", array($farminfo['id']));
- $Client = Client::Load($farminfo['clientid']);
-
- // Get AmazonEC2 Object
- $AmazonEC2Client = AmazonEC2::GetInstance(AWSRegions::GetAPIURL($farminfo['region']));
- $AmazonEC2Client->SetAuthKeys($Client->AWSPrivateKey, $Client->AWSCertificate);
-
- $farminfo = $this->db->GetRow("SELECT * FROM farms WHERE id=?", array($farminfo['id']));
- if ($farminfo['status'] != FARM_STATUS::RUNNING)
- {
- $this->logger->warn("[FarmID: {$farminfo['id']}] Farm terminated. There is no need to scale it.");
- return;
- }
-
- foreach ($farm_roles as $farm_ami)
- {
- $DBFarmRole = DBFarmRole::LoadByID($farm_ami['id']);
-
- if ($DBFarmRole->ReplaceToAMI != '')
- {
- $this->logger->warn("[FarmID: {$farminfo['id']}] Role '{$DBFarmRole->GetRoleName()}' being synchronized. This role will not be scalled.");
- continue;
- }
-
- // Get polling interval in seconds
- $polling_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_POLLING_INTERVAL)*60;
- $dt_last_polling = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_LAST_POLLING_TIME);
- if ($dt_last_polling && $dt_last_polling+$polling_interval > time())
- {
- $this->logger->info("Polling interval: every {$polling_interval} seconds");
- continue;
- }
-
- // Set Last polling time
- $DBFarmRole->SetSetting(DBFarmRole::SETTING_SCALING_LAST_POLLING_TIME, time());
- // Get current count of running and pending instances.
- $this->logger->info(sprintf("Processing role '%s'", $DBFarmRole->GetRoleName()));
-
- $RoleScalingManager = new RoleScalingManager($DBFarmRole);
- foreach ($RoleScalingManager->GetEnabledAlgos() as $ScalingAlgo)
- {
- $this->logger->info(sprintf("Checking %s scaling algorithm...", get_class($ScalingAlgo)));
-
- $res = $ScalingAlgo->MakeDecision($DBFarmRole);
- $this->logger->info(sprintf("%s result: %s", get_class($ScalingAlgo), $res));
-
- if ($res == ScalingAlgo::STOP_SCALING)
- {
- exit();
- }
- if ($res == ScalingAlgo::NOOP)
- {
- //TODO:
- }
- elseif ($res == ScalingAlgo::DOWNSCALE)
- {
- /*
- Timeout instance's count decrease. Decreases instance?s count after scaling
- resolution ?the spare instances are running? for selected timeout interval
- from scaling EditOptions
- */
-
- // We have to check timeout limits before new scaling (downscaling) process will be initiated
- if($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_TIMEOUT_ENABLED))
- { // if the farm timeout is exceeded
- // checking timeout interval.
-
- $last_down_scale_data_time = strtotime($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_DATETIME, false));
- $timeout_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_DOWNSCALE_TIMEOUT);
-
- // check the time interval to continue scaling or cancel it...
- if(time() - $last_down_scale_data_time < $timeout_interval*60)
- {
- // if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
- $this->logger->info(new FarmLogMessage($farminfo['id'],
- sprintf("The running time is too small to terminate any instance in farm %s, role %s",
- $farminfo['name'],
- $instanceinfo['role_name']
- )
- ));
- continue;
- }
- } // end Timeout instance's count decrease
- $sort = ($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_KEEP_OLDEST) == 1) ? 'DESC' : 'ASC';
-
- $instances = $this->db->GetAll("SELECT * FROM farm_instances WHERE state = ? AND farm_roleid=? ORDER BY dtadded {$sort}",
- array(INSTANCE_STATE::RUNNING, $DBFarmRole->ID)
- );
-
- $got_valid_instance = false;
-
- // Select instance that will be terminated
- //
- // * Instances ordered by uptime (oldest wil be choosen)
- // * Instance cannot be mysql master
- // * Choose the one that was rebundled recently
- while (!$got_valid_instance && count($instances) > 0)
- {
- $item = array_shift($instances);
- $instanceinfo = $item;
-
- // Exclude db master
- if ($instanceinfo["isdbmaster"] != 1)
- {
- /*
- * We do not want to delete the most recently synced instance. Because of LA fluctuation.
- * I.e. LA may skyrocket during sync and drop dramatically after sync.
- */
- if ($instanceinfo["dtlastsync"] != 0)
- {
- $chk_sync_time = $this->db->GetOne("SELECT id FROM farm_instances
- WHERE dtlastsync > {$instanceinfo['dtlastsync']}
- AND farm_roleid='{$instanceinfo['farm_roleid']}' AND state != '".INSTANCE_STATE::TERMINATED."'");
- if ($chk_sync_time)
- $got_valid_instance = true;
- }
- else
- $got_valid_instance = true;
- }
- }
-
- if ($instanceinfo && $got_valid_instance)
- {
- $this->logger->info(sprintf("Instance '%s' selected for termination...", $instanceinfo['instance_id']));
- $allow_terminate = false;
-
-
-
- // Shutdown an instance just before a full hour running
- $response = $AmazonEC2Client->DescribeInstances($instanceinfo['instance_id']);
- if ($response && $response->reservationSet->item)
- {
- $launch_time = strtotime($response->reservationSet->item->instancesSet->item->launchTime);
- $time = 3600 - (time() - $launch_time) % 3600;
-
- // Terminate instance in < 10 minutes for full hour.
- if ($time <= 600)
- $allow_terminate = true;
- else
- {
- $timeout = round(($time - 600) / 60, 1);
- $this->logger->info(new FarmLogMessage($farminfo['id'], sprintf("Farm %s, role %s scaling down. Instance '%s' will be terminated in %s minutes. Launch time: %s",
- $farminfo['name'],
- $instanceinfo['role_name'],
- $instanceinfo['instance_id'],
- $timeout,
- $response->reservationSet->item->instancesSet->item->launchTime
- )));
- }
- }
- //
-
- if ($allow_terminate)
- {
- try
- {
- $this->logger->info(new FarmLogMessage($farminfo['id'],
- sprintf("Scheduled termination for instance %s (%s). It will be terminated in 3 minutes.",
- $instanceinfo["instance_id"],
- $instanceinfo["external_ip"]
- )
- ));
-
- Scalr::FireEvent($farminfo['id'], new BeforeHostTerminateEvent(DBInstance::LoadByID($instanceinfo['id'], false)));
- }
- catch (Exception $e)
- {
- $this->logger->fatal(sprintf("Cannot terminate %s: %s",
- $farminfo['id'],
- $instanceinfo['instance_id'],
- $e->getMessage()
- ));
- }
- }
- }
- else
- $this->logger->warn(sprintf("Scalr unable to determine what instance it should terminate. Skipping..."));
-
- break;
- }
- elseif ($res == ScalingAlgo::UPSCALE)
- {
- /*
- Timeout instance's count increase. Increases instance's count after
- scaling resolution ?need more instances? for selected timeout interval
- from scaling EditOptions
- */
-
- if($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_TIMEOUT_ENABLED))
- {
- // if the farm timeout is exceeded
- // checking timeout interval.
- $last_up_scale_data_time = strtotime($DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_DATETIME, false));
- $timeout_interval = $DBFarmRole->GetSetting(DBFarmRole::SETTING_SCALING_UPSCALE_TIMEOUT);
-
- // check the time interval to continue scaling or cancel it...
- if(time() - $last_up_scale_data_time < $timeout_interval*60)
- {
- // if the launch time is too small to terminate smth in this role -> go to the next role in foreach()
- $this->logger->info(new FarmLogMessage($farminfo['id'],
- sprintf("The last scaling time interval is too small to start a new instance in farm %s, role %s",
- $farminfo['name'],
- $instanceinfo['role_name']
- )
- ));
- continue;
- }
- }// end Timeout instance's count increase
-
- $farminfo = $this->db->GetRow("SELECT * FROM farms WHERE id=?", array($farminfo['id']));
- if ($farminfo['status'] != FARM_STATUS::RUNNING)
- {
- $this->logger->warn("[FarmID: {$farminfo['id']}] Farm terminated. There is no need to scale it.");
- break;
- }
-
- $instance_id = Scalr::RunInstance($DBFarmRole, false, false, true);
- if ($instance_id)
- $this->logger->info(new FarmLogMessage($farminfo['id'], sprintf("Starting new instance. InstanceID = %s.", $instance_id)));
-
- break;
- }
- }
- }
- }
- }