/lib/CloudForecast/Gearman/Worker.pm

https://github.com/onopm/cloudforecast · Perl · 277 lines · 235 code · 41 blank · 1 comment · 26 complexity · e62dec526898553faaecb4968b213a1f MD5 · raw file

  1. package CloudForecast::Gearman::Worker;
  2. use strict;
  3. use warnings;
  4. use base qw/Class::Accessor::Fast/;
  5. use Gearman::Worker;
  6. use UNIVERSAL::require;
  7. use Storable qw//;
  8. use Parallel::Prefork::SpareWorkers;
  9. use CloudForecast::Log;
  10. use CloudForecast::ConfigLoader;
  11. use CloudForecast::Gearman::Scoreboard;
  12. use CloudForecast::Ledge;
  13. #preload
  14. require CloudForecast::Data;
  15. __PACKAGE__->mk_accessors(qw/configloader
  16. restarter
  17. max_workers
  18. max_requests_per_child
  19. max_exection_time/);
  20. our $GEARMAN_WORKER_CONNECT = {};
  21. sub new {
  22. my $class = shift;
  23. my $args = ref $_[0] ? shift : { @_ };
  24. my $configloader = CloudForecast::ConfigLoader->new({
  25. root_dir => $args->{root_dir},
  26. global_config => $args->{global_config}
  27. });
  28. $configloader->load_global_config();
  29. my $global_config = $configloader->global_config;
  30. die 'gearman is disabled' unless $global_config->{gearman_enable};
  31. $class->SUPER::new({
  32. configloader => $configloader,
  33. restarter => $args->{restarter},
  34. max_workers => $args->{max_workers} || 4,
  35. max_requests_per_child => $args->{max_requests_per_child} || 50,
  36. max_exection_time => $args->{max_exection_time} || 60,
  37. });
  38. }
  39. sub gearman_worker {
  40. my $self = shift;
  41. my $global_config = $self->configloader->global_config;
  42. my $host = $global_config->{gearman_server}->{host};
  43. my $port = $global_config->{gearman_server}->{port} || 7003;
  44. my $worker = $GEARMAN_WORKER_CONNECT->{"${host}:$port"};
  45. return $worker if $worker;
  46. $worker = Gearman::Worker->new;
  47. $worker->job_servers( "${host}:$port" );
  48. $GEARMAN_WORKER_CONNECT->{"${host}:$port"} = $worker;
  49. $worker;
  50. }
  51. sub load_resource {
  52. my $self = shift;
  53. my $args = shift;
  54. my $resource_class = $args->{resource_class};
  55. die "resource_class not defined" unless $resource_class;
  56. $resource_class = ucfirst $resource_class;
  57. $resource_class = "CloudForecast::Data::" . $resource_class;
  58. $resource_class->require or die $@;
  59. my $resource = $resource_class->new({
  60. hostname => $args->{hostname},
  61. address => $args->{address},
  62. details => $args->{details},
  63. args => $args->{args},
  64. component_config => $args->{component_config},
  65. global_config => $self->configloader->global_config,
  66. });
  67. return $resource;
  68. }
  69. sub load_ledge {
  70. my $self = shift;
  71. return $self->{_ledge} if $self->{_ledge};
  72. $self->{_ledge} = CloudForecast::Ledge->new(
  73. data_dir => $self->configloader->global_config->{data_dir},
  74. db_name => $self->configloader->global_config->{db_name},
  75. );
  76. $self->{_ledge};
  77. }
  78. sub fetcher_worker {
  79. my $self = shift;
  80. my $worker = $self->gearman_worker;
  81. $worker->register_function('fetcher', sub {
  82. my $job = shift;
  83. my $resource;
  84. eval {
  85. my $args;
  86. eval {
  87. $args = Storable::thaw($job->arg);
  88. $args or die "invalid arg";
  89. };
  90. die "failed thaw: $@" if $@;
  91. $resource = $self->load_resource($args);
  92. $resource->exec_fetch;
  93. };
  94. CloudForecast::Log->warn("fetcher failed: $@") if $@;
  95. 1;
  96. } );
  97. $self->run_worker(@_);
  98. }
  99. sub updater_worker {
  100. my $self = shift;
  101. my $worker = $self->gearman_worker;
  102. $worker->register_function('updater', sub {
  103. my $job = shift;
  104. eval {
  105. my $args;
  106. eval {
  107. $args = Storable::thaw($job->arg);
  108. $args or die "invalid arg";
  109. };
  110. die "failed thaw: $@" if $@;
  111. my $resource = $self->load_resource($args);
  112. $resource->exec_updater($args->{result});
  113. };
  114. CloudForecast::Log->warn("fetcher failed: $@") if $@;
  115. 1;
  116. });
  117. $worker->register_function('ledge', sub {
  118. my $job = shift;
  119. my $ret = {
  120. error => 0,
  121. };
  122. eval {
  123. my $method;
  124. my @args;
  125. eval {
  126. my $args = Storable::thaw($job->arg);
  127. $args or die 'invalid arg';
  128. @args = @$args;
  129. $method = shift @args;
  130. $method or die 'no method';
  131. $method =~ m!^(set|add|get|delete|expire)$! or die "invalid method: $method";
  132. };
  133. die "failed thaw: $@" if $@;
  134. my $ledge = $self->load_ledge();
  135. if ( $method =~ m!^(set|add|delete|expire)! ) {
  136. eval {
  137. $ret->{status} = $ledge->can($method)->($ledge, @args);
  138. };
  139. if ( $@ ) {
  140. $ret->{error} = 1;
  141. $ret->{errorstr} = $@;
  142. }
  143. }
  144. else {
  145. eval {
  146. ( $ret->{data}, $ret->{csum} ) = $ledge->get(@args);
  147. };
  148. if ( $@ ) {
  149. $ret->{error} = 1;
  150. $ret->{errorstr} = $@;
  151. }
  152. }
  153. };
  154. if ( $@ ) {
  155. CloudForecast::Log->warn("ledge failed: $@");
  156. $ret->{error} = 1;
  157. $ret->{errorstr} = $@;
  158. }
  159. Storable::nfreeze($ret);
  160. });
  161. $self->run_worker(@_);
  162. }
  163. sub watchdog_zombie {
  164. my $self = shift;
  165. my $scoreboard = shift;
  166. my $pid = fork();
  167. die "fork failed: $!" unless defined $pid;
  168. return $pid if($pid); # main process
  169. $0 = "$0 (watchdog)";
  170. while ( 1 ) {
  171. my @statuses = $scoreboard->get_parsed_statuses;
  172. for my $status ( @statuses ) {
  173. if ( $status->{status} eq CloudForecast::Gearman::Scoreboard::STATUS_ACTIVE
  174. && time - $status->{time} > $self->max_exection_time ) {
  175. CloudForecast::Log->warn("exection_time exceed, kill: " . $status->{pid});
  176. kill 'TERM', $status->{pid}
  177. }
  178. if ( $status->{status} eq CloudForecast::Gearman::Scoreboard::STATUS_IDLE
  179. && time - $status->{time} > 360 + int(30*(0.5-rand(1))) ) {
  180. CloudForecast::Log->warn("idle_time exceed, kill: " . $status->{pid});
  181. kill 'TERM', $status->{pid}
  182. }
  183. }
  184. sleep 30;
  185. }
  186. exit;
  187. }
  188. sub run_worker {
  189. my $self = shift;
  190. my $scoreboard = CloudForecast::Gearman::Scoreboard->new(
  191. undef, $self->max_workers );
  192. my @watchdog_pid;
  193. push @watchdog_pid, $self->watchdog_zombie( $scoreboard );
  194. if ( $self->restarter ) {
  195. CloudForecast::Log->debug("restarter start");
  196. push @watchdog_pid, $self->configloader->watchdog;
  197. }
  198. my $pm = Parallel::Prefork::SpareWorkers->new({
  199. max_workers => $self->max_workers,
  200. min_spare_workers => $self->max_workers,
  201. scoreboard => $scoreboard,
  202. trap_signals => {
  203. TERM => 'TERM',
  204. HUP => 'TERM',
  205. USR1 => undef,
  206. }
  207. });
  208. while ( $pm->signal_received ne 'TERM' ) {
  209. $pm->start and next;
  210. $0 = "$0 (worker)";
  211. my $i = 0;
  212. my $worker = $self->gearman_worker;
  213. $worker->work(
  214. on_start => sub {
  215. $i++;
  216. $pm->set_status( CloudForecast::Gearman::Scoreboard::STATUS_ACTIVE );
  217. },
  218. on_fail => sub {
  219. $pm->set_status( CloudForecast::Gearman::Scoreboard::STATUS_IDLE );
  220. },
  221. on_complete => sub {
  222. $pm->set_status( CloudForecast::Gearman::Scoreboard::STATUS_IDLE );
  223. },
  224. stop_if => sub {
  225. $i++ >= $self->max_requests_per_child
  226. }
  227. );
  228. $pm->finish;
  229. }
  230. $pm->wait_all_children();
  231. for my $pid ( @watchdog_pid ) {
  232. kill 'TERM', $pid;
  233. waitpid( $pid, 0 );
  234. }
  235. }
  236. 1;