PageRenderTime 183ms CodeModel.GetById 3ms app.highlight 169ms RepoModel.GetById 1ms app.codeStats 1ms

/lib/MHA/ServerManager.pm

https://github.com/luismottacampos/mha4mysql-manager
Perl | 1438 lines | 1272 code | 114 blank | 52 comment | 173 complexity | eec8d260c4b3bc04477c9e3fba19669e MD5 | raw file
   1#!/usr/bin/env perl
   2
   3#  Copyright (C) 2011 DeNA Co.,Ltd.
   4#
   5#  This program is free software; you can redistribute it and/or modify
   6#  it under the terms of the GNU General Public License as published by
   7#  the Free Software Foundation; either version 2 of the License, or
   8#  (at your option) any later version.
   9#
  10#  This program is distributed in the hope that it will be useful,
  11#  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13#  GNU General Public License for more details.
  14#
  15#  You should have received a copy of the GNU General Public License
  16#   along with this program; if not, write to the Free Software
  17#  Foundation, Inc.,
  18#  51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  19
  20package MHA::ServerManager;
  21
  22use strict;
  23use warnings FATAL => 'all';
  24use Carp qw(croak);
  25use English qw(-no_match_vars);
  26use MHA::SlaveUtil;
  27use MHA::DBHelper;
  28use MHA::Server;
  29use MHA::ManagerConst;
  30use Parallel::ForkManager;
  31
  32sub new {
  33  my $class = shift;
  34  my $self  = {
  35    servers          => [],
  36    dead_servers     => [],
  37    alive_servers    => [],
  38    alive_slaves     => [],
  39    failed_slaves    => [],
  40    latest_slaves    => [],
  41    oldest_slaves    => [],
  42    unmanaged_slaves => [],
  43    orig_master      => undef,
  44    new_master       => undef,
  45    logger           => undef,
  46    @_,
  47  };
  48  return bless $self, $class;
  49}
  50
  51sub set_servers($$) {
  52  my $self        = shift;
  53  my $servers_ref = shift;
  54  $self->{servers} = $servers_ref;
  55}
  56
  57sub set_latest_slaves($$) {
  58  my $self        = shift;
  59  my $servers_ref = shift;
  60  $self->{latest_slaves} = $servers_ref;
  61}
  62
  63sub set_oldest_slaves($$) {
  64  my $self        = shift;
  65  my $servers_ref = shift;
  66  $self->{oldest_slaves} = $servers_ref;
  67}
  68
  69sub set_unmanaged_slaves($$) {
  70  my $self        = shift;
  71  my $servers_ref = shift;
  72  $self->{unmanaged_slaves} = $servers_ref;
  73}
  74
  75sub get_servers($) {
  76  my $self = shift;
  77  return @{ $self->{servers} };
  78}
  79
  80sub get_dead_servers($) {
  81  my $self = shift;
  82  return @{ $self->{dead_servers} };
  83}
  84
  85sub get_alive_servers($) {
  86  my $self = shift;
  87  return @{ $self->{alive_servers} };
  88}
  89
  90sub get_alive_slaves($) {
  91  my $self = shift;
  92  return @{ $self->{alive_slaves} };
  93}
  94
  95sub get_failed_slaves($) {
  96  my $self = shift;
  97  return @{ $self->{failed_slaves} };
  98}
  99
 100sub get_latest_slaves($) {
 101  my $self = shift;
 102  return @{ $self->{latest_slaves} };
 103}
 104
 105sub get_oldest_slaves($) {
 106  my $self = shift;
 107  return @{ $self->{oldest_slaves} };
 108}
 109
 110sub get_unmanaged_slaves($) {
 111  my $self = shift;
 112  return @{ $self->{unmanaged_slaves} };
 113}
 114
 115sub add_dead_server($$) {
 116  my $self   = shift;
 117  my $server = shift;
 118  push @{ $self->{dead_servers} }, $server;
 119}
 120
 121sub add_alive_server($$) {
 122  my $self   = shift;
 123  my $server = shift;
 124  push @{ $self->{alive_servers} }, $server;
 125}
 126
 127sub add_alive_slave($$) {
 128  my $self   = shift;
 129  my $server = shift;
 130  push @{ $self->{alive_slaves} }, $server;
 131}
 132
 133sub add_failed_slave($$) {
 134  my $self   = shift;
 135  my $server = shift;
 136  push @{ $self->{failed_slaves} }, $server;
 137}
 138
 139sub add_unmanaged_slave($$) {
 140  my $self   = shift;
 141  my $server = shift;
 142  push @{ $self->{unmanaged_slaves} }, $server;
 143}
 144
 145sub set_orig_master($$) {
 146  my $self   = shift;
 147  my $server = shift;
 148  $self->{orig_master}   = $server;
 149  $server->{orig_master} = 1;
 150}
 151
 152sub get_orig_master($) {
 153  my $self = shift;
 154  return $self->{orig_master};
 155}
 156
 157sub init_servers($) {
 158  my $self    = shift;
 159  my $log     = $self->{logger};
 160  my @servers = $self->get_servers();
 161  $self->{dead_servers}     = [];
 162  $self->{alive_servers}    = [];
 163  $self->{alive_slaves}     = [];
 164  $self->{failed_slaves}    = [];
 165  $self->{unmanaged_slaves} = [];
 166  foreach my $server (@servers) {
 167
 168    if ( $server->{dead} ) {
 169      $self->add_dead_server($server);
 170    }
 171    elsif ( $server->{unmanaged} ) {
 172      $self->add_unmanaged_slave($server);
 173    }
 174    else {
 175      $self->add_alive_server($server);
 176      if ( $server->{not_slave} eq '0' && !$server->{orig_master} ) {
 177        if ( !$server->is_sql_thread_error() && !$server->{lack_relay_log} ) {
 178          $self->add_alive_slave($server);
 179        }
 180        else {
 181          $self->add_failed_slave($server);
 182        }
 183      }
 184    }
 185  }
 186  my @alive_servers = $self->get_alive_servers();
 187  if ( $#alive_servers <= -1 ) {
 188    $log->error("There is no alive server. We can't do failover");
 189    croak;
 190  }
 191  my @alive_slaves = $self->get_alive_slaves();
 192  if ( $#alive_slaves <= -1 ) {
 193    $log->error("There is no alive slave. We can't do failover");
 194    croak;
 195  }
 196}
 197
 198sub set_logger($$) {
 199  my $self   = shift;
 200  my $logger = shift;
 201  $self->{logger} = $logger;
 202}
 203
 204sub connect_all_and_read_server_status($$$$) {
 205  my $self             = shift;
 206  my $dead_master_host = shift;
 207  my $dead_master_ip   = shift;
 208  my $dead_master_port = shift;
 209  my $log              = $self->{logger};
 210  my @servers          = $self->get_servers();
 211  $log->debug("Connecting to servers..");
 212
 213  my $should_die         = 0;
 214  my $connection_checker = new Parallel::ForkManager( $#servers + 1 );
 215  $connection_checker->run_on_start(
 216    sub {
 217      my ( $pid, $target ) = @_;
 218    }
 219  );
 220  $connection_checker->run_on_finish(
 221    sub {
 222      my ( $pid, $exit_code, $target ) = @_;
 223      if ( $exit_code == $MHA::ManagerConst::MYSQL_DEAD_RC ) {
 224        $target->{dead} = 1;
 225      }
 226      elsif ($exit_code) {
 227        $should_die = 1;
 228      }
 229    }
 230  );
 231  foreach my $target (@servers) {
 232    unless ( $target->{logger} ) {
 233      $target->{logger} = $log;
 234    }
 235    $connection_checker->start($target) and next;
 236    eval {
 237      $SIG{INT} = $SIG{HUP} = $SIG{QUIT} = $SIG{TERM} = "DEFAULT";
 238      if ( $dead_master_host
 239        && $dead_master_ip
 240        && $dead_master_port )
 241      {
 242        if (
 243          $target->server_equals(
 244            $dead_master_host, $dead_master_ip, $dead_master_port
 245          )
 246          )
 247        {
 248          $connection_checker->finish($MHA::ManagerConst::MYSQL_DEAD_RC);
 249        }
 250      }
 251      my $rc = $target->connect_check(2);
 252      $connection_checker->finish($rc);
 253    };
 254    if ($@) {
 255      $log->error($@);
 256      undef $@;
 257      $connection_checker->finish(1);
 258    }
 259    $connection_checker->finish(0);
 260  }
 261  $connection_checker->wait_all_children;
 262  if ($should_die) {
 263    $log->error("Got fatal error, stopping operations");
 264    croak;
 265  }
 266
 267  foreach my $target (@servers) {
 268    next if ( $target->{dead} );
 269    $target->connect_and_get_status();
 270  }
 271  $self->init_servers();
 272  $self->compare_slave_version();
 273  $log->debug("Connecting to servers done.");
 274  $self->validate_current_master();
 275}
 276
 277sub get_oldest_version($) {
 278  my $self    = shift;
 279  my @servers = $self->get_alive_servers();
 280  my $oldest_version;
 281  foreach my $server (@servers) {
 282    if ( $server->{oldest_major_version} ) {
 283      $oldest_version = $server->{mysql_version};
 284      last;
 285    }
 286  }
 287  return $oldest_version;
 288}
 289
 290sub compare_slave_version($) {
 291  my $self    = shift;
 292  my @servers = $self->get_alive_servers();
 293  my $log     = $self->{logger};
 294  $log->debug(" Comparing MySQL versions..");
 295  my $min_major_version;
 296  foreach (@servers) {
 297    my $dbhelper = $_->{dbhelper};
 298    next if ( $_->{dead} || $_->{not_slave} );
 299    my $parsed_major_version =
 300      MHA::NodeUtil::parse_mysql_major_version( $_->{mysql_version} );
 301    if (!$min_major_version
 302      || $parsed_major_version < $min_major_version )
 303    {
 304      $min_major_version = $parsed_major_version;
 305    }
 306  }
 307  foreach (@servers) {
 308    my $dbhelper = $_->{dbhelper};
 309    next if ( $_->{dead} || $_->{not_slave} );
 310    my $parsed_major_version =
 311      MHA::NodeUtil::parse_mysql_major_version( $_->{mysql_version} );
 312    if ( $min_major_version == $parsed_major_version ) {
 313      $_->{oldest_major_version} = 1;
 314    }
 315    else {
 316      $_->{oldest_major_version} = 0;
 317    }
 318  }
 319  $log->debug("  Comparing MySQL versions done.");
 320}
 321
 322sub print_filter_rules($$) {
 323  my $self   = shift;
 324  my $master = shift;
 325  my $log    = $self->{logger};
 326  my $msg    = "Bad Binlog/Replication filtering rules:\n";
 327  $msg .= $master->print_filter(1) if ( $master && !$master->{dead} );
 328  my @slaves = $self->get_alive_slaves();
 329  foreach my $slave (@slaves) {
 330    $msg .= $slave->print_filter();
 331  }
 332  $log->warning($msg);
 333}
 334
 335sub validate_repl_filter($$) {
 336  my $self   = shift;
 337  my $master = shift;
 338  my $log    = $self->{logger};
 339
 340  $log->info("Checking replication filtering settings..");
 341
 342  my $binlog_do_db;
 343  my $binlog_ignore_db;
 344
 345  # If master is alive
 346  if ( $master && !$master->{dead} ) {
 347    $binlog_do_db     = $master->{Binlog_Do_DB};
 348    $binlog_ignore_db = $master->{Binlog_Ignore_DB};
 349    $log->info(
 350      " binlog_do_db= $binlog_do_db, binlog_ignore_db= $binlog_ignore_db");
 351  }
 352
 353  my @slaves = $self->get_alive_slaves();
 354  my $replicate_do_db;
 355  my $replicate_ignore_db;
 356  my $replicate_do_table;
 357  my $replicate_ignore_table;
 358  my $replicate_wild_do_table;
 359  my $replicate_wild_ignore_table;
 360  foreach (@slaves) {
 361    $replicate_do_db = $_->{Replicate_Do_DB} unless ($replicate_do_db);
 362    $replicate_ignore_db = $_->{Replicate_Ignore_DB}
 363      unless ($replicate_ignore_db);
 364    $replicate_do_table = $_->{Replicate_Do_Table} unless ($replicate_do_table);
 365    $replicate_ignore_table = $_->{Replicate_Ignore_Table}
 366      unless ($replicate_ignore_table);
 367    $replicate_wild_do_table = $_->{Replicate_Wild_Do_Table}
 368      unless ($replicate_wild_do_table);
 369    $replicate_wild_ignore_table = $_->{Replicate_Wild_Ignore_Table}
 370      unless ($replicate_wild_ignore_table);
 371    if ( $_->{log_bin} ) {
 372      $binlog_do_db     = $_->{Binlog_Do_DB}     unless ($binlog_do_db);
 373      $binlog_ignore_db = $_->{Binlog_Ignore_DB} unless ($binlog_ignore_db);
 374    }
 375    if ( $replicate_do_db ne $_->{Replicate_Do_DB}
 376      || $replicate_ignore_db ne $_->{Replicate_Ignore_DB}
 377      || $replicate_do_table ne $_->{Replicate_Do_Table}
 378      || $replicate_ignore_table ne $_->{Replicate_Ignore_Table}
 379      || $replicate_wild_do_table ne $_->{Replicate_Wild_Do_Table}
 380      || $replicate_wild_ignore_table ne $_->{Replicate_Wild_Ignore_Table} )
 381    {
 382      $log->error(
 383        sprintf(
 384"Replication filtering check failed on %s! All slaves must have same replication filtering rules. Check SHOW SLAVE STATUS output and set my.cnf correctly.",
 385          $_->get_hostinfo() )
 386      );
 387      $self->print_filter_rules($master);
 388      return 1;
 389    }
 390    if ( $_->{log_bin} ) {
 391      if ( $binlog_do_db ne $_->{Binlog_Do_DB}
 392        || $binlog_ignore_db ne $_->{Binlog_Ignore_DB} )
 393      {
 394        $log->error(
 395          sprintf(
 396"Binlog filtering check failed on %s! All log-bin enabled servers must have same binlog filtering rules (same binlog-do-db and binlog-ignore-db). Check SHOW MASTER STATUS output and set my.cnf correctly.",
 397            $_->get_hostinfo() )
 398        );
 399        $self->print_filter_rules($master);
 400        return 1;
 401      }
 402    }
 403  }
 404  if ( $binlog_do_db && $replicate_do_db ) {
 405    if ( $binlog_do_db ne $replicate_do_db ) {
 406      $log->error(
 407        sprintf(
 408"binlog_do_db on master(%s) must be the same as replicate_do_db on slaves(%s).",
 409          $binlog_do_db, $replicate_do_db
 410        )
 411      );
 412      $self->print_filter_rules($master);
 413      return 1;
 414    }
 415  }
 416  if ( $binlog_ignore_db && $replicate_ignore_db ) {
 417    if ( $binlog_ignore_db ne $replicate_ignore_db ) {
 418      $log->error(
 419        sprintf(
 420"binlog_ignore_db on master(%s) must be the same as replicate_ignore_db on slaves(%s).",
 421          $binlog_ignore_db, $replicate_ignore_db
 422        )
 423      );
 424      $self->print_filter_rules($master);
 425      return 1;
 426    }
 427  }
 428  $log->info(" Replication filtering check ok.");
 429  return 0;
 430}
 431
 432sub validate_num_alive_servers($$$) {
 433  my $self              = shift;
 434  my $current_master    = shift;
 435  my $ignore_fail_check = shift;
 436  my $log               = $self->{logger};
 437  my @dead_servers      = $self->get_dead_servers();
 438  my @failed_slaves     = $self->get_failed_slaves();
 439
 440  foreach (@dead_servers) {
 441    next if ( $_->{id} eq $current_master->{id} );
 442    next if ( $ignore_fail_check && $_->{ignore_fail} );
 443    $log->error(
 444      sprintf( " Server %s is dead, but must be alive! Check server settings.",
 445        $_->get_hostinfo() )
 446    );
 447    croak;
 448  }
 449  foreach (@failed_slaves) {
 450    next if ( $ignore_fail_check && $_->{ignore_fail} );
 451    $log->error(
 452      sprintf( " Replication on %s fails! Check server settings.",
 453        $_->get_hostinfo() )
 454    );
 455    croak;
 456  }
 457
 458  return 0;
 459}
 460
 461# Check the following
 462# 1. All slaves are read_only (INFO)
 463# 2. All slaves see the same master ip/port (ERROR)
 464# 3. All slaves set relay_log_purge=0 (WARN)
 465# 4. All slaves have same replication filter rules with a master (ERROR)
 466# return 0: ok, others: NG
 467sub validate_slaves($$$) {
 468  my $self              = shift;
 469  my $check_repl_filter = shift;
 470  my $master            = shift;
 471  my $log               = $self->{logger};
 472  my @slaves            = $self->get_alive_slaves();
 473  my ( $mip, $mport ) = ();
 474  my $error = 0;
 475  $log->info("Checking slave configurations..");
 476
 477  foreach (@slaves) {
 478    if ( $_->{read_only} ne '1' ) {
 479      $log->info(
 480        sprintf( " read_only=1 is not set on slave %s.\n", $_->get_hostinfo() )
 481      );
 482    }
 483    if ( $_->{relay_purge} ne '0' ) {
 484      $log->warning(
 485        sprintf( " relay_log_purge=0 is not set on slave %s.\n",
 486          $_->get_hostinfo() )
 487      );
 488    }
 489    if ( $_->{log_bin} eq '0' ) {
 490      $log->warning(
 491        sprintf(
 492          " log-bin is not set on slave %s. This host can not be a master.\n",
 493          $_->get_hostinfo() )
 494      );
 495    }
 496  }
 497  $error = $self->validate_repl_filter($master)
 498    if ($check_repl_filter);
 499  return $error;
 500}
 501
 502sub get_alive_server_by_ipport {
 503  my $self = shift;
 504  my $ip   = shift;
 505  my $port = shift;
 506  $self->get_server_by_ipport( $ip, $port, 1 );
 507}
 508
 509sub get_server_by_ipport {
 510  my $self       = shift;
 511  my $ip         = shift;
 512  my $port       = shift;
 513  my $alive_only = shift;
 514  my @servers;
 515  if ($alive_only) {
 516    @servers = $self->get_alive_servers();
 517  }
 518  else {
 519    @servers = $self->get_servers();
 520  }
 521  foreach (@servers) {
 522    if ( $_->{ip} eq $ip && $_->{port} == $port ) {
 523      return $_;
 524    }
 525  }
 526  return;
 527}
 528
 529sub get_alive_server_by_hostport {
 530  my $self    = shift;
 531  my $host    = shift;
 532  my $port    = shift;
 533  my @servers = $self->get_alive_servers();
 534  foreach (@servers) {
 535    if ( $_->{hostname} eq $host && $_->{port} == $port ) {
 536      return $_;
 537    }
 538  }
 539  return;
 540}
 541
 542sub get_server_from_by_id {
 543  my $self        = shift;
 544  my $servers_ref = shift;
 545  my $id          = shift;
 546  my @servers     = @$servers_ref;
 547  foreach (@servers) {
 548    if ( $_->{id} eq $id ) {
 549      return $_;
 550    }
 551  }
 552  return;
 553}
 554
 555sub get_alive_server_by_id {
 556  my $self          = shift;
 557  my $id            = shift;
 558  my @alive_servers = $self->get_alive_servers();
 559  foreach (@alive_servers) {
 560    if ( $_->{id} eq $id ) {
 561      return $_;
 562    }
 563  }
 564  return;
 565}
 566
 567sub get_alive_slave_by_id {
 568  my $self         = shift;
 569  my $id           = shift;
 570  my @alive_slaves = $self->get_alive_slaves();
 571  foreach (@alive_slaves) {
 572    if ( $_->{id} eq $id ) {
 573      return $_;
 574    }
 575  }
 576  return;
 577}
 578
 579sub get_master_by_slave {
 580  my $self  = shift;
 581  my $slave = shift;
 582  return $self->get_server_by_ipport( $slave->{Master_IP},
 583    $slave->{Master_Port} );
 584}
 585
 586sub validate_current_master($) {
 587  my $self          = shift;
 588  my $log           = $self->{logger};
 589  my @alive_servers = $self->get_alive_servers();
 590  my %master_hash;
 591  my $num_slaves        = 0;
 592  my $not_slave_servers = 0;
 593  foreach (@alive_servers) {
 594    if ( $_->{not_slave} eq '0' ) {
 595      $master_hash{"$_->{Master_IP}:$_->{Master_Port}"} = $_;
 596      $num_slaves++;
 597    }
 598    else {
 599      $not_slave_servers++;
 600    }
 601  }
 602
 603  if ( $not_slave_servers >= 2 ) {
 604    $log->error(
 605"There are $not_slave_servers non-slave servers! MHA manages at most one non-slave server. Check configurations."
 606    );
 607    croak;
 608  }
 609
 610  if ( $num_slaves < 1 ) {
 611    $log->error(
 612      "There is not any alive slave! Check slave settings for details.");
 613    croak;
 614  }
 615
 616  # verify masters exist in a config file
 617  my $master;
 618  foreach my $key ( keys(%master_hash) ) {
 619    my $slave = $master_hash{$key};
 620    $master = $self->get_master_by_slave($slave);
 621    unless ($master) {
 622      $log->error(
 623        sprintf(
 624"Master %s:%d from which slave %s replicates is not defined in the configuration file!",
 625          $slave->{Master_IP}, $slave->{Master_Port},
 626          $slave->get_hostinfo()
 627        )
 628      );
 629      croak;
 630    }
 631  }
 632
 633  my $real_master;
 634  if ( keys(%master_hash) >= 2 ) {
 635    $real_master = $self->get_primary_master( \%master_hash );
 636  }
 637  else {
 638    $real_master = $master;
 639    $self->set_orig_master($real_master);
 640  }
 641  $self->validate_master_ip_port($real_master);
 642  return $real_master;
 643}
 644
 645sub validate_master_ip_port {
 646  my $self                 = shift;
 647  my $real_master          = shift;
 648  my $log                  = $self->{logger};
 649  my $has_unmanaged_slaves = 0;
 650  my @alive_servers        = $self->get_alive_servers();
 651  foreach my $slave (@alive_servers) {
 652    next if ( $slave->{id} eq $real_master->{id} );
 653    unless ( $self->get_alive_slave_by_id( $slave->{id} ) ) {
 654      $log->error(
 655        sprintf( "Server %s is alive, but does not work as a slave!",
 656          $slave->get_hostinfo() )
 657      );
 658      croak;
 659    }
 660    if (
 661      !(
 662           ( $slave->{Master_IP} eq $real_master->{ip} )
 663        && ( $slave->{Master_Port} == $real_master->{port} )
 664      )
 665      )
 666    {
 667      if ( $slave->{multi_tier_slave} ) {
 668        $slave->{unmanaged} = 1;
 669        $has_unmanaged_slaves = 1;
 670      }
 671      else {
 672        my $msg = sprintf(
 673          "Slave %s replicates from %s:%d, but real master is %s!",
 674          $slave->get_hostinfo(), $slave->{Master_Host},
 675          $slave->{Master_Port},  $real_master->get_hostinfo()
 676        );
 677        $log->error($msg);
 678        croak;
 679      }
 680    }
 681  }
 682  if ($has_unmanaged_slaves) {
 683    $self->init_servers();
 684  }
 685}
 686
 687sub get_multi_master_print_info {
 688  my $self            = shift;
 689  my $master_hash_ref = shift;
 690  my %master_hash     = %$master_hash_ref;
 691  my $str             = "";
 692  foreach my $key ( keys(%master_hash) ) {
 693    my $slave  = $master_hash{$key};
 694    my $master = $self->get_master_by_slave($slave);
 695    $str .= "Master " . $master->get_hostinfo();
 696    $str .=
 697", replicating from $master->{Master_Host}($master->{Master_IP}:$master->{Master_Port})"
 698      if ( $master->{Master_Host} );
 699    $str .= ", read-only" if ( $master->{read_only} );
 700    $str .= ", dead"      if ( $master->{dead} );
 701    $str .= "\n";
 702  }
 703  $str .= "\n";
 704  return $str;
 705}
 706
 707sub get_primary_master {
 708  my $self            = shift;
 709  my $master_hash_ref = shift;
 710  my $log             = $self->{logger};
 711  my @alive_servers   = $self->get_alive_servers();
 712  my %master_hash     = %$master_hash_ref;
 713
 714  my $num_real_masters = 0;
 715  my $real_master;
 716  foreach my $key ( keys(%master_hash) ) {
 717    my $slave  = $master_hash{$key};
 718    my $master = $self->get_master_by_slave($slave);
 719    next if ( !$master->{dead} && $master->{read_only} );
 720    $real_master = $master;
 721    $num_real_masters++;
 722  }
 723  if ( $num_real_masters < 1 ) {
 724    $log->error(
 725      sprintf(
 726"Multi-master configuration is detected, but all of them are read-only! Check configurations for details. Master configurations are as below: \n%s",
 727        $self->get_multi_master_print_info($master_hash_ref) )
 728    );
 729    croak;
 730  }
 731  elsif ( $num_real_masters >= 2 ) {
 732    $log->error(
 733      sprintf(
 734"Multi-master configuration is detected, but two or more masters are either writable (read-only is not set) or dead! Check configurations for details. Master configurations are as below: \n%s",
 735        $self->get_multi_master_print_info($master_hash_ref) )
 736    );
 737    croak;
 738  }
 739  else {
 740    $self->set_orig_master($real_master);
 741    $log->info(
 742      sprintf(
 743"Multi-master configuration is detected. Current primary(writable) master is %s",
 744        $real_master->get_hostinfo() )
 745    );
 746    $log->info(
 747      sprintf( "Master configurations are as below: \n%s",
 748        $self->get_multi_master_print_info($master_hash_ref) )
 749    );
 750    $self->init_servers();
 751  }
 752  return $real_master;
 753}
 754
 755sub get_candidate_masters($) {
 756  my $self        = shift;
 757  my $log         = $self->{logger};
 758  my @servers     = $self->get_servers();
 759  my @ret_servers = ();
 760  foreach (@servers) {
 761    next if ( $_->{dead} eq '1' );
 762    if ( $_->{candidate_master} >= 1 ) {
 763      push( @ret_servers, $_ );
 764    }
 765  }
 766  return @ret_servers;
 767}
 768
 769sub print_dead_servers {
 770  my $self = shift;
 771  $self->print_servers( $self->{dead_servers} );
 772}
 773
 774sub print_alive_servers {
 775  my $self          = shift;
 776  my $log           = $self->{logger};
 777  my @alive_servers = $self->get_alive_servers();
 778  foreach (@alive_servers) {
 779    $log->info( "  " . $_->get_hostinfo() );
 780  }
 781}
 782
 783sub print_alive_slaves {
 784  my $self = shift;
 785  $self->print_servers( $self->{alive_slaves} );
 786}
 787
 788sub print_latest_slaves {
 789  my $self = shift;
 790  $self->print_servers( $self->{latest_slaves} );
 791}
 792
 793sub print_oldest_slaves {
 794  my $self = shift;
 795  $self->print_servers( $self->{oldest_slaves} );
 796}
 797
 798sub print_failed_slaves_if {
 799  my $self          = shift;
 800  my $log           = $self->{logger};
 801  my @failed_slaves = $self->get_failed_slaves();
 802  if ( $#failed_slaves >= 0 ) {
 803    $log->info("Failed Slaves:");
 804    $self->print_servers( $self->{failed_slaves} );
 805  }
 806}
 807
 808sub print_unmanaged_slaves_if {
 809  my $self             = shift;
 810  my $log              = $self->{logger};
 811  my @unmanaged_slaves = $self->get_unmanaged_slaves();
 812  if ( $#unmanaged_slaves >= 0 ) {
 813    $log->info("Unmanaged Servers:");
 814    $self->print_servers( $self->{unmanaged_slaves} );
 815  }
 816}
 817
 818sub print_servers {
 819  my ( $self, $servers_ref ) = @_;
 820  my @servers = @$servers_ref;
 821  foreach (@servers) {
 822    $_->print_server();
 823  }
 824}
 825
 826sub disconnect_all($) {
 827  my $self    = shift;
 828  my $log     = $self->{logger};
 829  my @servers = $self->get_alive_servers();
 830  foreach (@servers) {
 831    $_->disconnect();
 832  }
 833}
 834
 835# Check master is not reachable from all alive slaves
 836# prerequisite: all slaves see the same master
 837# return 0;ok 1: running
 838sub is_master_reachable_from_slaves($$) {
 839  my $self       = shift;
 840  my $slaves_ref = shift;
 841  my $log        = $self->{logger};
 842  my @slaves     = $self->get_alive_slaves();
 843  $log->info("Checking the current master is not reachable from all slaves..");
 844  foreach (@slaves) {
 845    my $dbhelper = $_->{dbhelper};
 846    $dbhelper->stop_io_thread();
 847    $dbhelper->start_io_thread();
 848    sleep(3);
 849    my %status = $dbhelper->check_slave_status();
 850    if ( $status{Status} ne '0' || !defined( $status{Slave_IO_Running} ) ) {
 851      $log->error(
 852        sprintf( "Got error when stopping/starting io thread on %s",
 853          $_->get_hostinfo() )
 854      );
 855      return 1;
 856    }
 857    if ( $status{Slave_IO_Running} eq "Yes" ) {
 858      $log->warning(
 859        sprintf( "Master is reachable from slave %s", $_->get_hostinfo() ) );
 860      return 1;
 861    }
 862    $dbhelper->stop_io_thread();
 863    $log->info(
 864      sprintf( " Master is not reachable from slave %s", $_->get_hostinfo() ) );
 865  }
 866  $log->info("  done.");
 867  return 0;
 868}
 869
 870# checking slave status again before starting main operations.
 871# alive slaves info was already fetched by connect_all_and_read_server_status,
 872# so check_slave_status should not fail here. If it fails, we die here.
 873sub read_slave_status($) {
 874  my $self   = shift;
 875  my $log    = $self->{logger};
 876  my @slaves = $self->get_alive_slaves();
 877
 878  $log->debug("Fetching current slave status..");
 879  foreach (@slaves) {
 880    my $dbhelper  = $_->{dbhelper};
 881    my ($sstatus) = ();
 882    my %status    = $dbhelper->check_slave_status();
 883
 884    # This should not happen so die if it happens
 885    if ( $status{Status} ) {
 886      my $msg = "Checking slave status failed.";
 887      $msg .= " err=$status{Errstr}" if ( $status{Errstr} );
 888      $log->error($msg);
 889      croak;
 890    }
 891
 892    $_->{latest}                = 0;
 893    $_->{Master_Log_File}       = $status{Master_Log_File};
 894    $_->{Read_Master_Log_Pos}   = $status{Read_Master_Log_Pos};
 895    $_->{Relay_Master_Log_File} = $status{Relay_Master_Log_File};
 896    $_->{Exec_Master_Log_Pos}   = $status{Exec_Master_Log_Pos};
 897    $_->{Relay_Log_File}        = $status{Relay_Log_File};
 898    $_->{Relay_Log_Pos}         = $status{Relay_Log_Pos};
 899  }
 900  $log->debug(" Fetching current slave status done.");
 901}
 902
 903sub start_sql_threads_if($) {
 904  my $self   = shift;
 905  my @slaves = $self->get_alive_slaves();
 906  foreach my $slave (@slaves) {
 907    $slave->start_sql_thread_if();
 908  }
 909}
 910
 911sub get_failover_advisory_locks($) {
 912  my $self   = shift;
 913  my $log    = $self->{logger};
 914  my @slaves = $self->get_alive_slaves();
 915  foreach my $slave (@slaves) {
 916    if ( $slave->get_failover_advisory_lock() ) {
 917      $log->error(
 918        sprintf(
 919"Getting advisory lock failed on %s. Maybe failover script or purge_relay_logs script is running on the same slave?",
 920          $slave->get_hostinfo() )
 921      );
 922      croak;
 923    }
 924  }
 925}
 926
 927sub identify_latest_slaves($$) {
 928  my $self        = shift;
 929  my $find_oldest = shift;
 930  $find_oldest = 0 unless ($find_oldest);
 931  my $log    = $self->{logger};
 932  my @slaves = $self->get_alive_slaves();
 933  my @latest = ();
 934  foreach (@slaves) {
 935    my $a = $latest[0]{Master_Log_File};
 936    my $b = $latest[0]{Read_Master_Log_Pos};
 937    if (
 938      !$find_oldest
 939      && (
 940           ( !$a && !defined($b) )
 941        || ( $_->{Master_Log_File} gt $latest[0]{Master_Log_File} )
 942        || ( ( $_->{Master_Log_File} ge $latest[0]{Master_Log_File} )
 943          && $_->{Read_Master_Log_Pos} > $latest[0]{Read_Master_Log_Pos} )
 944      )
 945      )
 946    {
 947      @latest = ();
 948      push( @latest, $_ );
 949    }
 950    elsif (
 951      $find_oldest
 952      && (
 953           ( !$a && !defined($b) )
 954        || ( $_->{Master_Log_File} lt $latest[0]{Master_Log_File} )
 955        || ( ( $_->{Master_Log_File} le $latest[0]{Master_Log_File} )
 956          && $_->{Read_Master_Log_Pos} < $latest[0]{Read_Master_Log_Pos} )
 957      )
 958      )
 959    {
 960      @latest = ();
 961      push( @latest, $_ );
 962    }
 963    elsif ( ( $_->{Master_Log_File} eq $latest[0]{Master_Log_File} )
 964      && ( $_->{Read_Master_Log_Pos} == $latest[0]{Read_Master_Log_Pos} ) )
 965    {
 966      push( @latest, $_ );
 967    }
 968  }
 969  foreach (@latest) {
 970    $_->{latest} = 1 if ( !$find_oldest );
 971    $_->{oldest} = 1 if ($find_oldest);
 972  }
 973  $log->info(
 974    sprintf(
 975      "The %s binary log file/position on all slaves is" . " %s:%d\n",
 976      $find_oldest ? "oldest" : "latest", $latest[0]{Master_Log_File},
 977      $latest[0]{Read_Master_Log_Pos}
 978    )
 979  );
 980  if ($find_oldest) {
 981    $self->set_oldest_slaves( \@latest );
 982  }
 983  else {
 984    $self->set_latest_slaves( \@latest );
 985  }
 986}
 987
 988sub identify_oldest_slaves($) {
 989  my $self = shift;
 990  return $self->identify_latest_slaves(1);
 991}
 992
 993# 1: higher
 994# -1: older
 995# 0: equal
 996sub pos_cmp {
 997  my ( $self, $a_mlf, $a_mlp, $b_mlf, $b_mlp ) = @_;
 998  return 0 if ( $a_mlf eq $b_mlf && $a_mlp == $b_mlp );
 999  return -1 if ( $a_mlf lt $b_mlf || ( $a_mlf le $b_mlf && $a_mlp < $b_mlp ) );
1000  return 1;
1001}
1002
1003sub set_no_master_if_older($$$) {
1004  my $self   = shift;
1005  my $mlf    = shift;
1006  my $mlp    = shift;
1007  my @slaves = $self->get_alive_slaves();
1008  foreach (@slaves) {
1009    $_->{no_master} = 1
1010      if (
1011      $self->pos_cmp( $_->{Master_Log_File}, $_->{Read_Master_Log_Pos},
1012        $mlf, $mlp ) < 0
1013      );
1014  }
1015}
1016
1017sub get_oldest_limit_pos($) {
1018  my $self   = shift;
1019  my @slaves = $self->get_alive_slaves();
1020  my $target;
1021  foreach (@slaves) {
1022    next if ( $_->{ignore_fail} );
1023    my $a = $target->{Master_Log_File};
1024    my $b = $target->{Read_Master_Log_Pos};
1025    if (
1026         ( !$a && !defined($b) )
1027      || ( $_->{Master_Log_File} lt $target->{Master_Log_File} )
1028      || ( ( $_->{Master_Log_File} le $target->{Master_Log_File} )
1029        && $_->{Read_Master_Log_Pos} < $target->{Read_Master_Log_Pos} )
1030      )
1031    {
1032      $target = $_;
1033    }
1034  }
1035  return ( $target->{Master_Log_File}, $target->{Read_Master_Log_Pos} )
1036    if ($target);
1037}
1038
1039# check slave is too behind master or not
1040# 0: no or acceptable delay
1041# 1: unacceptable delay (can not be a master)
1042sub check_slave_delay($$$) {
1043  my $self   = shift;
1044  my $target = shift;
1045  my $latest = shift;
1046  my $log    = $self->{logger};
1047  $log->debug(
1048    sprintf( "Checking replication delay on %s.. ", $target->get_hostinfo() ) );
1049  if (
1050    ( $latest->{Master_Log_File} gt $target->{Relay_Master_Log_File} )
1051    || ( $latest->{Read_Master_Log_Pos} >
1052      $target->{Exec_Master_Log_Pos} + 100000000 )
1053    )
1054  {
1055    $log->warning(
1056      sprintf(
1057" Slave %s SQL Thread delays too much. Latest log file:%s:%d, Current log file:%s:%d. This server is not selected as a new master because recovery will take long time.\n",
1058        $target->get_hostinfo(),        $latest->{Master_Log_File},
1059        $latest->{Read_Master_Log_Pos}, $target->{Relay_Master_Log_File},
1060        $target->{Exec_Master_Log_Pos}
1061      )
1062    );
1063    return 1;
1064  }
1065  $log->debug(" ok.");
1066  return 0;
1067}
1068
1069# The following servers can not be master:
1070# - dead servers
1071# - Set no_master in conf files (i.e. DR servers)
1072# - log_bin is disabled
1073# - Major version is not the oldest
1074# - too much replication delay
1075sub get_bad_candidate_masters($$$) {
1076  my $self                    = shift;
1077  my $latest_slave            = shift;
1078  my $check_replication_delay = shift;
1079  my $log                     = $self->{logger};
1080
1081  my @servers     = $self->get_alive_slaves();
1082  my @ret_servers = ();
1083  foreach (@servers) {
1084    if (
1085         $_->{no_master} >= 1
1086      || $_->{log_bin} eq '0'
1087      || $_->{oldest_major_version} eq '0'
1088      || (
1089        $latest_slave
1090        && ( $check_replication_delay
1091          && $self->check_slave_delay( $_, $latest_slave ) >= 1 )
1092      )
1093      )
1094    {
1095      push( @ret_servers, $_ );
1096    }
1097  }
1098  return @ret_servers;
1099}
1100
1101sub is_target_bad_for_new_master {
1102  my $self   = shift;
1103  my $target = shift;
1104  my @bad    = $self->get_bad_candidate_masters();
1105  foreach (@bad) {
1106    return 1 if ( $target->{id} eq $_->{id} );
1107  }
1108  return 0;
1109}
1110
1111# Picking up new master
1112# If preferred node is specified, one of active preferred nodes will be new master.
1113# If the latest server behinds too much (i.e. stopping sql thread for online backups), we should not use it as a new master, but we should fetch relay log there. Even though preferred master is configured, it does not become a master if it's far behind.
1114sub select_new_master {
1115  my $self                    = shift;
1116  my $prio_new_master_host    = shift;
1117  my $prio_new_master_port    = shift;
1118  my $check_replication_delay = shift;
1119  $check_replication_delay = 1 if ( !defined($check_replication_delay) );
1120
1121  my $log    = $self->{logger};
1122  my @latest = $self->get_latest_slaves();
1123  my @slaves = $self->get_alive_slaves();
1124
1125  my @pref = $self->get_candidate_masters();
1126  my @bad =
1127    $self->get_bad_candidate_masters( $latest[0], $check_replication_delay );
1128
1129  if ( $prio_new_master_host && $prio_new_master_port ) {
1130    my $new_master =
1131      $self->get_alive_server_by_hostport( $prio_new_master_host,
1132      $prio_new_master_port );
1133    if ($new_master) {
1134      my $a = $self->get_server_from_by_id( \@bad, $new_master->{id} );
1135      unless ($a) {
1136        $log->info("$prio_new_master_host can be new master.");
1137        return $new_master;
1138      }
1139      else {
1140        $log->error("$prio_new_master_host is bad as a new master!");
1141        return;
1142      }
1143    }
1144    else {
1145      $log->error("$prio_new_master_host is not alive!");
1146      return;
1147    }
1148  }
1149
1150  $log->info("Searching new master from slaves..");
1151  $log->info(" Candidate masters from the configuration file:");
1152  $self->print_servers( \@pref );
1153  $log->info(" Non-candidate masters:");
1154  $self->print_servers( \@bad );
1155
1156  return $latest[0]
1157    if ( $#pref < 0 && $#bad < 0 && $latest[0]->{latest_priority} );
1158
1159  if ( $latest[0]->{latest_priority} ) {
1160    $log->info(
1161" Searching from candidate_master slaves which have received the latest relay log events.."
1162    ) if ( $#pref >= 0 );
1163    foreach my $h (@latest) {
1164      foreach my $p (@pref) {
1165        if ( $h->{id} eq $p->{id} ) {
1166          return $h
1167            if ( !$self->get_server_from_by_id( \@bad, $p->{id} ) );
1168        }
1169      }
1170    }
1171    $log->info("  Not found.") if ( $#pref >= 0 );
1172  }
1173
1174  #new master is not latest
1175  $log->info(" Searching from all candidate_master slaves..")
1176    if ( $#pref >= 0 );
1177  foreach my $s (@slaves) {
1178    foreach my $p (@pref) {
1179      if ( $s->{id} eq $p->{id} ) {
1180        my $a = $self->get_server_from_by_id( \@bad, $p->{id} );
1181        return $s unless ($a);
1182      }
1183    }
1184  }
1185  $log->info("  Not found.") if ( $#pref >= 0 );
1186
1187  if ( $latest[0]->{latest_priority} ) {
1188    $log->info(
1189" Searching from all slaves which have received the latest relay log events.."
1190    );
1191    foreach my $h (@latest) {
1192      my $a = $self->get_server_from_by_id( \@bad, $h->{id} );
1193      return $h unless ($a);
1194    }
1195    $log->info("  Not found.");
1196  }
1197
1198  # none of latest servers can not be a master
1199  $log->info(" Searching from all slaves..");
1200  foreach my $s (@slaves) {
1201    my $a = $self->get_server_from_by_id( \@bad, $s->{id} );
1202    return $s unless ($a);
1203  }
1204  $log->info("  Not found.");
1205
1206  return;
1207}
1208
1209sub get_new_master_binlog_position($$) {
1210  my $self     = shift;
1211  my $target   = shift;                 # master
1212  my $dbhelper = $target->{dbhelper};
1213  my $log      = $self->{logger};
1214  $log->info("Getting new master's binlog name and position..");
1215  my ( $file, $pos ) = $dbhelper->show_master_status();
1216  if ( $file && defined($pos) ) {
1217    $log->info(" $file:$pos");
1218    $log->info(
1219      sprintf(
1220" All other slaves should start replication from here. Statement should be: CHANGE MASTER TO MASTER_HOST='%s', MASTER_PORT=%d, MASTER_LOG_FILE='%s', MASTER_LOG_POS=%d, MASTER_USER='%s', MASTER_PASSWORD='xxx';",
1221        ( $target->{hostname} eq $target->{ip} )
1222        ? $target->{hostname}
1223        : ("$target->{hostname} or $target->{ip}"),
1224        $target->{port},
1225        $file,
1226        $pos,
1227        $target->{repl_user}
1228      )
1229    );
1230  }
1231  else {
1232    $log->error("Getting new master's binlog position failed!");
1233    return;
1234  }
1235  return ( $file, $pos );
1236}
1237
1238sub change_master_and_start_slave {
1239  my ( $self, $target, $master, $master_log_file, $master_log_pos, $log ) = @_;
1240  $log = $self->{logger} unless ($log);
1241  return if ( $target->{id} eq $master->{id} );
1242  my $dbhelper = $target->{dbhelper};
1243  $log->info(
1244    sprintf(
1245      " Resetting slave %s and starting replication from the new master %s..",
1246      $target->get_hostinfo(),
1247      $master->get_hostinfo()
1248    )
1249  );
1250  $target->stop_slave($log) unless ( $target->{not_slave} );
1251  $dbhelper->reset_slave()  unless ( $target->{not_slave} );
1252  $dbhelper->change_master( $target->{use_ip_for_change_master}
1253    ? $master->{ip}
1254    : $master->{hostname},
1255    $master->{port}, $master_log_file, $master_log_pos, $master->{repl_user},
1256    $master->{repl_password} );
1257  $log->info(" Executed CHANGE MASTER.");
1258
1259  # After executing CHANGE MASTER, relay_log_purge is automatically disabled.
1260  # If the original value is 0, we should turn to 0 explicitly.
1261  unless ( $target->{relay_purge} ) {
1262    $target->disable_relay_log_purge();
1263  }
1264  my $ret = $target->start_slave($log);
1265  unless ($ret) {
1266    $log->info(" Slave started.");
1267  }
1268  return $ret;
1269}
1270
1271sub get_current_alive_master($) {
1272  my $self   = shift;
1273  my $log    = $self->{logger};
1274  my $master = $self->get_orig_master();
1275  unless ($master) {
1276    $log->error(
1277      "MySQL master is not correctly configured. Check master/slave settings");
1278    croak;
1279  }
1280  my $m = $self->get_alive_server_by_id( $master->{id} );
1281  unless ($m) {
1282    $log->warning("MySQL master is not currently alive!");
1283    return;
1284  }
1285  $log->info( sprintf( "Current Alive Master: %s", $m->get_hostinfo() ) );
1286  return $master;
1287}
1288
1289sub stop_io_threads {
1290  my $self         = shift;
1291  my $log          = $self->{logger};
1292  my @alive_slaves = $self->get_alive_slaves();
1293  my $pm           = new Parallel::ForkManager( $#alive_slaves + 1 );
1294  foreach my $target (@alive_slaves) {
1295    $target->stop_io_thread($target);
1296    exit 0;
1297  }
1298  $pm->wait_all_children;
1299  return 0;
1300}
1301
1302sub check_repl_priv {
1303  my $self    = shift;
1304  my @servers = $self->get_alive_servers();
1305  foreach my $target (@servers) {
1306    $target->check_repl_priv();
1307  }
1308}
1309
1310sub release_failover_advisory_lock {
1311  my $self    = shift;
1312  my @servers = $self->get_alive_servers();
1313  foreach my $target (@servers) {
1314    $target->release_failover_advisory_lock();
1315  }
1316}
1317
1318sub get_current_servers_ascii {
1319  my $self         = shift;
1320  my $orig_master  = shift;
1321  my @alive_slaves = $self->get_alive_slaves();
1322
1323  my $str = "$orig_master->{hostname} (current master)";
1324  $str .= " ($orig_master->{node_label})"
1325    if ( $orig_master->{node_label} );
1326  $str .= "\n";
1327  foreach my $slave (@alive_slaves) {
1328    $str .= " +--" . "$slave->{hostname}";
1329    $str .= " ($slave->{node_label})" if ( $slave->{node_label} );
1330    $str .= "\n";
1331  }
1332  $str .= "\n";
1333  return $str;
1334}
1335
1336sub print_servers_ascii {
1337  my $self         = shift;
1338  my $orig_master  = shift;
1339  my $log          = $self->{logger};
1340  my @alive_slaves = $self->get_alive_slaves();
1341
1342  my $str = "\n";
1343  $str .= $self->get_current_servers_ascii($orig_master);
1344  $log->info($str);
1345}
1346
1347sub print_servers_migration_ascii {
1348  my $self                     = shift;
1349  my $orig_master              = shift;
1350  my $new_master               = shift;
1351  my $orig_master_is_new_slave = shift;
1352  my $log                      = $self->{logger};
1353  my @alive_slaves             = $self->get_alive_slaves();
1354
1355  my $str = "\n";
1356  $str .= "From:\n";
1357  $str .= $self->get_current_servers_ascii($orig_master);
1358
1359  $str .= "To:\n";
1360  $str .= "$new_master->{hostname} (new master)";
1361  $str .= " ($new_master->{node_label})"
1362    if ( $new_master->{node_label} );
1363  $str .= "\n";
1364  foreach my $slave (@alive_slaves) {
1365    next if ( $slave->{id} eq $new_master->{id} );
1366    $str .= " +--" . "$slave->{hostname}";
1367    $str .= " ($slave->{node_label})" if ( $slave->{node_label} );
1368    $str .= "\n";
1369  }
1370  if ($orig_master_is_new_slave) {
1371    $str .= " +--" . "$orig_master->{hostname}";
1372    $str .= " ($orig_master->{node_label})" if ( $orig_master->{node_label} );
1373    $str .= "\n";
1374  }
1375  $log->info($str);
1376}
1377
1378# for manual failover/switch only
1379sub manually_decide_new_master {
1380  my $self        = shift;
1381  my $orig_master = shift;
1382  my $new_master  = shift;
1383  my $log         = $self->{logger};
1384
1385  printf(
1386    "\nStarting master switch from %s to %s? (yes/NO): ",
1387    $orig_master->get_hostinfo(),
1388    $new_master->get_hostinfo()
1389  );
1390  my $ret = <STDIN>;
1391  chomp($ret);
1392  if ( lc($ret) !~ /^y/ ) {
1393    print "Continue? (yes/NO): ";
1394    $ret = <STDIN>;
1395    chomp($ret);
1396    if ( lc($ret) !~ /^y/ ) {
1397      $orig_master->{not_error} = 1;
1398      die "Not typed yes. Stopping.";
1399    }
1400    print "Enter new master host name: ";
1401    $ret = <STDIN>;
1402    chomp($ret);
1403    $new_master = $self->get_alive_server_by_hostport( $ret, 3306 );
1404
1405    if ( !$new_master ) {
1406      die "New server not found!\n";
1407    }
1408    printf "Master switch to %s. OK? (yes/NO): ", $new_master->get_hostinfo();
1409    $ret = <STDIN>;
1410    chomp($ret);
1411    die "Not typed yes. Stopping. \n" if ( lc($ret) !~ /^y/ );
1412  }
1413  return $new_master;
1414}
1415
1416sub check_replication_health {
1417  my $self                = shift;
1418  my $allow_delay_seconds = shift;
1419  $allow_delay_seconds = 1 unless ($allow_delay_seconds);
1420  my $log          = $self->{logger};
1421  my @alive_slaves = $self->get_alive_slaves();
1422  foreach my $target (@alive_slaves) {
1423    $log->info("Checking replication health on $target->{hostname}..");
1424    if ( !$target->current_slave_position() ) {
1425      $log->error("Getting slave status failed!");
1426      croak;
1427    }
1428    if ( $target->has_replication_problem($allow_delay_seconds) ) {
1429      $log->error(" failed!");
1430      croak;
1431    }
1432    else {
1433      $log->info(" ok.");
1434    }
1435  }
1436}
1437
14381;