oar-server 2.5.4-2+deb8u1 / etc / oar / oar_phoenix.pl

#!/usr/bin/perl
# OAR Script to check and automatically reboot Suspected nodes.
# This script is intended to be started periodically from cron
# for example:
#    */10 * * * *	root /usr/sbin/oar_phoenix
# It needs the nodes to be configured to set up themselves
# in the Alive state at boot time.
# !!! Work in progress: this script needs to be improved !!!
# - No locking!
# - No logging!
# - Timed out processes may still run!
# - Hard timeout is not yet implemented:
#     it should trig a call to Judas to send an e-mail
#     to the admin.

use strict;
use warnings;
use YAML;
use OAR::IO;


################### CUSTOMIZABLE PART #######################

# File where phoenix stores the states of the broken nodes
my $PHOENIX_DBFILE="/var/lib/oar/phoenix/oar_phoenix.db";

# Directory where logfiles are created in case of problems
# !! NOT YET IMPLEMENTED !!
my $PHOENIX_LOGDIR="/var/lib/oar/phoenix/";

# Command sent to reboot a node (first attempt)
#my $PHOENIX_SOFT_REBOOTCMD="ssh -p 6667 {NODENAME} oardodo reboot";
my $PHOENIX_SOFT_REBOOTCMD="echo 'Soft reboot command for {NODENAME}: PHOENIX_SOFT_REBOOTCMD not configured'";

# Timeout for a soft rebooted node to be considered hard rebootable
my $PHOENIX_SOFT_TIMEOUT=300;

# Command sent to rebopot a node (seond attempt)
#my $PHOENIX_HARD_REBOOTCMD="oardodo ipmitool -U USERID -P PASSW0RD -H {NODENAME}-mgt power off;sleep 2;oardodo ipmitool -U USERID -P PASSW0RD -H {NODENAME}-mgt power on";
my $PHOENIX_HARD_REBOOTCMD="echo 'Hard reboot command for {NODENAME}: PHOENIX_HARD_REBOOTCMD not configured'";

# Timeout (s) for a hard rebooted node to be considered really broken, then
# an email is sent
# !! NOT YET IMPLEMENTED!!
my $PHOENIX_HARD_TIMEOUT=300;

# Max number of simultaneous reboots (soft OR hard)
my $PHOENIX_MAX_REBOOTS=10;

# Timout (s) for unix commands
my $PHOENIX_CMD_TIMEOUT=15;

# Get the broken nodes list (SQL request to customize)
my $base = OAR::IO::connect();
#my $date=OAR::IO::get_date($base);
#my @db=OAR::IO::get_nodes_with_given_sql($base,"(state='Absent' AND (available_upto < $date OR available_upto = 0))  or state='Suspected'");
#my @broken_nodes=OAR::IO::get_nodes_with_given_sql($base,"state='Suspected' and network_address NOT IN (SELECT distinct(network_address) FROM resources where resource_id IN (SELECT resource_id  FROM assigned_resources WHERE assigned_resource_index = 'CURRENT')) and network_address != '6po'");
my @broken_nodes=OAR::IO::get_nodes_with_given_sql($base,"state='Suspected' and network_address != '6po'");
OAR::IO::disconnect($base);

################ END OF CUSTOMIZABLE PART ####################

# Send a unix command ant timeout if necessary
sub send_cmd($) {
  my $cmd=shift;
  eval {
    open(my $LOGFILE,">>$PHOENIX_LOGDIR/oar_phoenix.log") or die "can't open logfile into $PHOENIX_LOGDIR/ for writing!: $!";
    local $SIG{ALRM} = sub {die "alarm\n"};
    alarm $PHOENIX_CMD_TIMEOUT;
    my $res = `$cmd 2>&1`;
    print $LOGFILE $res ."\n";
    close($LOGFILE);
    alarm 0;
  };
  if ($@) {
    die unless $@ eq "alarm\n";
    return "Timed out!\n";
  }
}

# Load the DB file
sub load_db($) {
  my $file = shift;
  open(my $YAMLFILE,$file) or die "can't open db file $file for reading: $!";
  return YAML::LoadFile($YAMLFILE);
  close($YAMLFILE);
}

# Export DB to file
sub save_db($$) {
  my $file = shift;
  my $ref = shift;
  open (my $YAMLFILE,">$file") or die "can't open db file $file for writing: $!";
  print $YAMLFILE YAML::Dump($ref);
  close($YAMLFILE);
}

# Init DB file
sub init_db($) {
  my $file = shift;
  if (!-s $file) {
    my %empty_hash;
    save_db($file,\%empty_hash);
  }
}

# Remove nodes that are no longer broken from DB
sub clean_db($$) {
  my $db=shift;
  my $broken_nodes=shift;
  foreach my $node (keys %$db) {
    if (!grep {$_ eq $node} @broken_nodes) {
      delete($db->{$node});
      # TODO: add a log entry
    }
  }
}

# Get nodes to soft_reboot
sub get_nodes_to_soft_reboot($$) {
  my $db=shift;
  my $broken_nodes=shift;
  my $nodes;
  my $c=0;
  foreach my $node (@broken_nodes) {
    if (!defined($db->{$node})) {
      $c++;
      push (@$nodes,$node);
    }
    last if ($c>=$PHOENIX_MAX_REBOOTS);
  }
  return $nodes;
}


# Get nodes to hard_reboot
sub get_nodes_to_hard_reboot($$) {
  my $db=shift;
  my $broken_nodes=shift;
  my $nodes;
  my $c=0;
  foreach my $node (@broken_nodes) {
    if (defined($db->{$node})) {
      if (defined($db->{$node}->{"soft_reboot"})) {
        if (time() > $db->{$node}->{"soft_reboot"} + $PHOENIX_SOFT_TIMEOUT) {
          $c++;
          push (@$nodes,$node);
        }
        last if ($c>=$PHOENIX_MAX_REBOOTS);
      }
    }
  }
  return $nodes;
}

# Soft reboot nodes
sub soft_reboot_nodes($$) {
  my $db=shift;
  my $nodes=shift;
  my $cmd;
  my $res;
  foreach my $node (@$nodes) {
    $cmd=$PHOENIX_SOFT_REBOOTCMD;
    $cmd =~ s/\{NODENAME\}/$node/g;
    print "Soft rebooting the broken node $node\n"; 
    $db->{$node}={ 'soft_reboot' => time() };
    send_cmd($cmd);
  }
}

# Hard reboot nodes
sub hard_reboot_nodes($$) {
  my $db=shift;
  my $nodes=shift;
  my $cmd;
  my $res;
  foreach my $node (@$nodes) {
    $cmd=$PHOENIX_HARD_REBOOTCMD;
    $cmd =~ s/\{NODENAME\}/$node/g;
    print "Hard rebooting the broken node $node\n"; 
    delete($db->{$node});
    $db->{$node}={ 'hard_reboot' => time() };
    $res=send_cmd($cmd);
  }
}

init_db($PHOENIX_DBFILE);
my $db=load_db($PHOENIX_DBFILE);
clean_db($db,@broken_nodes);
my $nodes=get_nodes_to_soft_reboot($db,@broken_nodes);
soft_reboot_nodes($db,$nodes);
$nodes=get_nodes_to_hard_reboot($db,@broken_nodes);
hard_reboot_nodes($db,$nodes);
save_db($PHOENIX_DBFILE,$db);