/usr/lib/oar/oar_meta

#!/usr/bin/perl
# $Id$

use strict;
use warnings;
use DBI();
use OAR::IO;
use OAR::Modules::Judas qw(oar_debug oar_warn oar_error set_current_log_category);
use OAR::Conf qw(init_conf dump_conf get_conf is_conf get_conf_with_default_param);
use Data::Dumper;
use OAR::Tools;
use OAR::Modules::Hulot;
use OAR::Schedulers::GanttHoleStorage;

# Log category
set_current_log_category('scheduler');

my $exit_code = 0;
my $base = OAR::IO::connect();
my $base_ro = OAR::IO::connect_ro();

init_conf($ENV{OARCONFFILE});
my $Order_part = get_conf("SCHEDULER_RESOURCE_ORDER");

my $binpath;
if (defined($ENV{OARDIR})){
    $binpath = $ENV{OARDIR}."/";
}else{
    die("[MetaSched] Error: OARDIR env variable must be defined\n");
}

my $Cpuset_field = get_conf("JOB_RESOURCE_MANAGER_PROPERTY_DB_FIELD");
my $Remote_host = get_conf("SERVER_HOSTNAME");
my $Remote_port = get_conf("SERVER_PORT");

my @Sched_available_suspended_resource_type;
my $sched_available_suspended_resource_type_tmp = get_conf("SCHEDULER_AVAILABLE_SUSPENDED_RESOURCE_TYPE");
if (!defined($sched_available_suspended_resource_type_tmp)){
    push(@Sched_available_suspended_resource_type, "default");
}else{
    @Sched_available_suspended_resource_type = split(" ",$sched_available_suspended_resource_type_tmp);
}

# Look at resources that we must add for each job
my $Resources_to_always_add_type = get_conf("SCHEDULER_RESOURCES_ALWAYS_ASSIGNED_TYPE");
my @Resources_to_always_add = ();

# Do we log every scheduler's computation ?
my $Log_scheduling = get_conf("SCHEDULER_LOG_DECISIONS");
if (defined($Log_scheduling) and $Log_scheduling ne "yes") { $Log_scheduling = undef; }

#minimum of seconds between each jobs
my $Security_time_overhead = 1;
if (is_conf("SCHEDULER_JOB_SECURITY_TIME")){
    $Security_time_overhead = get_conf("SCHEDULER_JOB_SECURITY_TIME");
}

my $Openssh_cmd = get_conf("OPENSSH_CMD");
$Openssh_cmd = OAR::Tools::get_default_openssh_cmd() if (!defined($Openssh_cmd));
if (is_conf("OAR_SSH_CONNECTION_TIMEOUT")){
    OAR::Tools::set_ssh_timeout(get_conf("OAR_SSH_CONNECTION_TIMEOUT"));
}
my $Server_hostname = get_conf("SERVER_HOSTNAME");
my $Deploy_hostname = get_conf("DEPLOY_HOSTNAME");
if (!defined($Deploy_hostname)){
    $Deploy_hostname = $Server_hostname;
}
my $Cosystem_hostname = get_conf("COSYSTEM_HOSTNAME");
if (!defined($Cosystem_hostname)){
    $Cosystem_hostname = $Server_hostname;
}
if (is_conf("OAR_RUNTIME_DIRECTORY")){
    OAR::Tools::set_default_oarexec_directory(get_conf("OAR_RUNTIME_DIRECTORY"));
}

#minimum of seconds to be considered like a hole in the gantt
my $Minimum_hole_time = 0;
if (is_conf("SCHEDULER_GANTT_HOLE_MINIMUM_TIME")){
    $Minimum_hole_time = get_conf("SCHEDULER_GANTT_HOLE_MINIMUM_TIME");
}

# waiting time when a reservation has not all of its nodes
my $Reservation_waiting_timeout = 300;

# global variables : initialized in init_scheduler function
my %besteffort_resource_occupation;

# stock the job ids that where already send to almighty
my %To_launch_jobs_already_treated;

my $current_time_sec = 0;
my $current_time_sql = "0000-00-00 00:00:00";

my $resa_admin_waiting_timeout = get_conf("RESERVATION_WAITING_RESOURCES_TIMEOUT");

oar_debug("[MetaSched] Starting Meta Scheduler\n");

###########################################################################################################
#Initialize Gantt tables with scheduled reservation jobs, Running jobs, toLaunch jobs and Launching jobs; #
###########################################################################################################

# First get resources that we must add for each reservation jobs
if (defined($Resources_to_always_add_type)){
    my $tmp_result_state_resources = OAR::IO::get_specific_resource_states($base,$Resources_to_always_add_type);
    if (defined($tmp_result_state_resources->{"Alive"})){
        @Resources_to_always_add = @{$tmp_result_state_resources->{"Alive"}};
        oar_debug("[MetaSched] Got resources to automatically add to every reservations: @Resources_to_always_add\n");
    }
}

$Reservation_waiting_timeout = $resa_admin_waiting_timeout if (defined($resa_admin_waiting_timeout));

# Take care of the currently (or nearly) running jobs
# Lock to prevent bipbip update in same time
OAR::IO::lock_table($base,["jobs","assigned_resources","gantt_jobs_predictions","gantt_jobs_resources","job_types","moldable_job_descriptions","resources","job_state_logs","gantt_jobs_predictions_log","gantt_jobs_resources_log"]);

#calculate now date with no overlap with other jobs
my $previous_ref_time_sec = OAR::IO::get_gantt_date($base);
$current_time_sec = OAR::IO::get_date($base);
if ($current_time_sec < $previous_ref_time_sec){
    # The system is very fast!!!
    my $tmp_delta = $previous_ref_time_sec - $current_time_sec;
    if ($tmp_delta > 60){
        oar_error("[OAR::Schedulers::Scheduler] init_scheduler : The previous scheduler step was performed $tmp_delta seconds in the future. It is bad, maybe the system clock of the DB server has changed. This can results in bad scheduling decisions.\n");
    }else{
        $current_time_sec = $previous_ref_time_sec;
        if ($tmp_delta > 1){
            # Avoid race condition that could increment too much the current time indefinitely
            oar_debug("[OAR::Schedulers::Scheduler] init_scheduler : The previous scheduler step was performed $tmp_delta seconds in the future. So we have to wait or the scheduling decisions will be bad\n");
            sleep($tmp_delta);
        }
    }
}
$current_time_sec++;
$current_time_sql = OAR::IO::local_to_sql($current_time_sec);

my ($Alive_resource_id_vec, undef) = OAR::IO::get_resource_ids_in_state($base,"Alive");
my ($Absent_resource_id_vec, undef) = OAR::IO::get_resource_ids_in_state($base,"Absent");
my ($Suspected_resource_id_vec, undef) = OAR::IO::get_resource_ids_in_state($base,"Suspected");
my ($Dead_resource_id_vec, @Dead_resource_ids) = OAR::IO::get_resource_ids_in_state($base,"Dead");

oar_debug("[MetaSched] Retrieve information for already scheduler reservations from database before flush (keep assign resources)\n");
my $reservation_already_there = OAR::IO::get_waiting_reservations_already_scheduled($base);
OAR::IO::gantt_flush_tables($base, $reservation_already_there, $Log_scheduling);
OAR::IO::set_gantt_date($base,$current_time_sec);

#Init the gantt chart with all resources
oar_debug("[MetaSched] Initialize the gantt structure\n");
my ($Max_resources, $All_resource_list_vec, $Only_default_type_resource_vec) = OAR::IO::get_vecs_resources($base);
my $Gantt = {};
$Gantt->{0}->{""}->{""}->{""} = OAR::Schedulers::GanttHoleStorage::new($Max_resources, $Minimum_hole_time);
OAR::Schedulers::GanttHoleStorage::add_new_resources($Gantt->{0}->{""}->{""}->{""}, $All_resource_list_vec);

oar_debug("[MetaSched] Begin processing of already scheduled reservations (accepted with resources assigned)\n");
# Add already scheduled reservations into the gantt
foreach my $i (keys(%{$reservation_already_there})){
    my $container_id = 0;
    my $inner_id = 0;
    my $placeholder_name = "";
    my $allowed_name = "";
    my $timesharing_user = "";
    my $timesharing_name = "";
    my $vec = '';
    my $types = OAR::IO::get_job_types_hash($base,$i);
    my $job_there = OAR::IO::get_job($base,$i);
    foreach my $r (@{$reservation_already_there->{$i}->{resources}}){
        vec($vec, $r, 1) = 1;
    }
    if (defined($types->{inner}) and ($types->{inner} =~ /^(\d+)$/)){
        $inner_id = $1;
        if (defined($Gantt->{$inner_id}->{""}->{""}->{""})){
            oar_debug("[MetaSched] [$i] inner job: using container $inner_id\n");
        }else{
            oar_debug("[MetaSched] [$i] inner job: container $inner_id does not exist but job is running, use container 0.\n");
            $inner_id = 0;
        }
    }
    if (defined($types->{container})){ # A container job cannot be placeholder or allowed or timesharing. 
        oar_debug("[MetaSched] [$i] Job is ($inner_id,,,)\n");
        $container_id = $i;
        oar_debug("[MetaSched] [$i] container job: create gantt ($container_id,,,)\n");
        $Gantt->{$container_id}->{""}->{""}->{""} = 
            OAR::Schedulers::GanttHoleStorage::new_with_1_hole($Max_resources, $Minimum_hole_time, $reservation_already_there->{$i}->{start_time}, $reservation_already_there->{$i}->{walltime} + $Security_time_overhead , $vec, $All_resource_list_vec);
    } else {
        ($placeholder_name, $allowed_name, $timesharing_user, $timesharing_name) = 
            OAR::Schedulers::GanttHoleStorage::manage_gantt_for_timesharing_and_placeholder($Gantt, $job_there->{job_user}, $job_there->{job_name}, $types, $inner_id, "[MetaSched] [$i]");
    }

    #Fill all other gantts
    OAR::Schedulers::GanttHoleStorage::fill_gantts($Gantt, $reservation_already_there->{$i}->{start_time}, $reservation_already_there->{$i}->{walltime} + $Security_time_overhead, $vec, $inner_id ,$placeholder_name, $allowed_name, $timesharing_user, $timesharing_name, "[MetaSched] [$i]"); 
}
oar_debug("[MetaSched] End processing of already scheduled reservations\n");

oar_debug("[MetaSched] Begin processing of current jobs\n");
my @initial_jobs = OAR::IO::get_jobs_in_multiple_states($base, ["Running","toLaunch","Launching","Finishing","Suspended","Resuming"]);
foreach my $job (@initial_jobs){
    my $container_id = 0;
    my $inner_id = 0;
    my $placeholder_name = "";
    my $allowed_name = "";
    my $timesharing_user = "";
    my $timesharing_name = "";
    my $i = $job->{job_id};

    next if ($job->{assigned_moldable_job} == 0);
    my $mold = OAR::IO::get_current_moldable_job($base,$job->{assigned_moldable_job});
    # The list of resources on which the job is running
    my ($resource_list_vec, @resource_list) = OAR::IO::get_job_current_resources($base, $job->{assigned_moldable_job},undef);

    my $date ;
    if ($job->{start_time} == 0) {
        $date = $current_time_sec;
    }elsif ($job->{start_time} + $mold->{moldable_walltime} < $current_time_sec){
        $date = $current_time_sec - $mold->{moldable_walltime};
    }else{
        $date = $job->{start_time};
    }
    oar_debug("[MetaSched] [$i] add job in database\n");
    OAR::IO::add_gantt_scheduled_jobs($base,$job->{assigned_moldable_job},$date,\@resource_list);

    my $types = OAR::IO::get_job_types_hash($base,$job->{job_id});
    # Ignore besteffort jobs
    if (! defined($types->{besteffort})){
        my $job_duration = $mold->{moldable_walltime};
        if ($job->{state} eq "Suspended"){
            # Remove resources of the type specified in SCHEDULER_AVAILABLE_SUSPENDED_RESOURCE_TYPE
            ($resource_list_vec, @resource_list) = OAR::IO::get_job_current_resources($base, $job->{assigned_moldable_job},\@Sched_available_suspended_resource_type);
        }
        if ($job->{suspended} eq "YES"){
            # This job was suspended so we must recalculate the walltime
            $job_duration += OAR::IO::get_job_suspended_sum_duration($base,$job->{job_id},$current_time_sec);
        }

        if (defined($types->{inner}) and ($types->{inner} =~ /^(\d+)$/)){
            $inner_id = $1;
            if (defined($Gantt->{$inner_id}->{""}->{""}->{""})){
                oar_debug("[MetaSched] [$i] inner job: using container $inner_id\n");
            }else{
                oar_debug("[MetaSched] [$i] inner job: container $inner_id does not exist but job is running, use container 0.\n");
                $inner_id = 0;
            }
        }
        if (defined($types->{container})){ # A container job cannot be placeholder or allowed or timesharing. 
            oar_debug("[MetaSched] [$i] Job is ($inner_id,,,)\n");
            $container_id = $i;
            oar_debug("[MetaSched] [$i] container job: create gantt ($container_id,,,)\n");
            $Gantt->{$container_id}->{""}->{""}->{""} = 
                OAR::Schedulers::GanttHoleStorage::new_with_1_hole($Max_resources, $Minimum_hole_time, $date, $job_duration + $Security_time_overhead, $resource_list_vec, $All_resource_list_vec);
        } else {
            ($placeholder_name, $allowed_name, $timesharing_user, $timesharing_name) = 
            OAR::Schedulers::GanttHoleStorage::manage_gantt_for_timesharing_and_placeholder($Gantt, $job->{job_user}, $job->{job_name}, $types, $inner_id, "[MetaSched] [$i]");
        }

        #Fill all other gantts
        OAR::Schedulers::GanttHoleStorage::fill_gantts($Gantt, $date, $job_duration + $Security_time_overhead, $resource_list_vec, $inner_id ,$placeholder_name, $allowed_name, $timesharing_user, $timesharing_name, "[MetaSched] [$i]"); 
    }else{
        #Stock information about besteffort jobs
        foreach my $r (@resource_list){
            $besteffort_resource_occupation{$r} = $job;
        }
    }
}
OAR::IO::unlock_table($base);
oar_debug("[MetaSched] End processing of current jobs\n");

# handle accepted advance reservations which do not have assigned resources, mostly because the prediction tables were completely flushed from outside of OAR
oar_debug("[MetaSched] Begin processing of the accepted reservations without assigned resources\n");
my @Rjobs = OAR::IO::get_waiting_reservation_jobs($base);
foreach my $job (@Rjobs){
    next if (defined($reservation_already_there->{$job->{job_id}}));

    my $container_id = 0;
    my $inner_id = 0;
    my $placeholder_name = "";
    my $allowed_name = "";
    my $timesharing_user = "";
    my $timesharing_name = "";
    my $i = $job->{job_id};

    my $types = OAR::IO::get_job_types_hash($base,$i);

    my $job_descriptions = OAR::IO::get_resources_data_structure_current_job($base,$job->{job_id});
    # For reservation we take the first moldable job
    my $moldable = $job_descriptions->[0];

    # Get the list of resources where the reservation will be able to be launched = resources in state Alive, Absent or Suspected
    # It is assumed here that Absent and Suspected resources could become Alive by the time of the start of the advance reservation job
    my $available_resources_vector = $Alive_resource_id_vec | $Absent_resource_id_vec | $Suspected_resource_id_vec;
    if (defined($types->{inner}) and ($types->{inner} =~ /^(\d+)$/)){
        $inner_id = $1;
        if (defined($Gantt->{$inner_id}->{""}->{""}->{""})){
            oar_debug("[MetaSched] [$i] inner job: using container $inner_id\n");
        }else{
            oar_debug("[MetaSched] [$i] inner job: container $inner_id does not exist.\n");
            OAR::IO::set_job_message($base,$i,"Container $inner_id does not exist");
            OAR::IO::set_job_scheduler_info($base,$i,"Container $inner_id does not exist");
            $inner_id=0;
            next;
        }
    }
    if (defined($types->{container})){ # A container job cannot be placeholder or allowed or timesharing. 
        oar_debug("[MetaSched] [$i] Job is ($inner_id,,,) and is a container (ignoring any timesharing or placeholder)\n");
    } else {
        ($placeholder_name, $allowed_name, $timesharing_user, $timesharing_name) =
            OAR::Schedulers::GanttHoleStorage::manage_gantt_for_timesharing_and_placeholder($Gantt, $job->{job_user}, $job->{job_name}, $types, $inner_id, "[MetaSched] [$i]");
    }

    my $free_resources_vec = OAR::Schedulers::GanttHoleStorage::get_free_resources($Gantt->{$inner_id}->{$allowed_name}->{$timesharing_user}->{$timesharing_name}, 
                                                                        $job->{start_time},
                                                                        $moldable->[1] + $Security_time_overhead,
                                                                   );
    $available_resources_vector &= $free_resources_vec;
    
    my $job_properties = "\'1\'";
    if ((defined($job->{properties})) and ($job->{properties} ne "")){
        $job_properties = $job->{properties};
    }

    my $resource_id_used_list_vector = '';
    my @tree_list;
    foreach my $m (@{$moldable->[0]}){
        my $tmp_properties = "\'1\'";
        if ((defined($m->{property})) and ($m->{property} ne "")){
            $tmp_properties = $m->{property};
        }
        my $tmp_tree;
        $tmp_tree = OAR::IO::get_possible_wanted_resources($base_ro,$available_resources_vector,$resource_id_used_list_vector,\@Dead_resource_ids,"$job_properties AND $tmp_properties", $m->{resources}, "".$Order_part);
        $tmp_tree = OAR::Schedulers::ResourceTree::delete_tree_nodes_with_not_enough_resources_and_unnecessary_subtrees($tmp_tree, undef);
        push(@tree_list, $tmp_tree);
        my @leafs = OAR::Schedulers::ResourceTree::get_tree_leafs($tmp_tree);
        foreach my $l (@leafs){
            vec($resource_id_used_list_vector, OAR::Schedulers::ResourceTree::get_current_resource_value($l), 1) = 1;
        }
    }
   
    my @resources;
    foreach my $t (@tree_list){
        foreach my $r (OAR::Schedulers::ResourceTree::get_tree_leafs($t)){
            push(@resources, OAR::Schedulers::ResourceTree::get_current_resource_value($r));
        }
    }

    if ($#resources >= 0){
        # We can schedule the job
        my $vec = '';
        foreach my $r (@resources){
            vec($vec, $r, 1) = 1;
        }

        # Create gantt for the new container
        if (defined($types->{container})){
            $container_id = $i;
            oar_debug("[MetaSched] [$i] container job: create gantt ($container_id,,,)\n");
            $Gantt->{$container_id}->{""}->{""}->{""} = OAR::Schedulers::GanttHoleStorage::new_with_1_hole($Max_resources, $Minimum_hole_time, $job->{start_time}, $moldable->[1] + $Security_time_overhead, $vec, $All_resource_list_vec);
        }

        #Fill all other gantts
        OAR::Schedulers::GanttHoleStorage::fill_gantts($Gantt, $job->{start_time}, $moldable->[1] + $Security_time_overhead, $vec, $inner_id, $placeholder_name, $allowed_name, $timesharing_user, $timesharing_name, "[MetaSched] [$i]"); 

        # Update database
        push(@resources, @Resources_to_always_add);
        oar_debug("[MetaSched] [$i] add job in database\n");
        oar_warn("[MetaSched] [$i] resources for reservation were lost, got them back\n");
        OAR::IO::add_gantt_scheduled_jobs($base,$moldable->[2],$job->{start_time},\@resources);
        OAR::IO::add_new_event($base,"GET_RESOURCES_BACK_FOR_RESERVATION",$job->{job_id},"Resources for reservation were lost, got them back");
        OAR::IO::set_job_message($base,$job->{job_id},"Renewed resources for reservation");
    }else{
        oar_warn("[MetaSched] [$i] Resources for reservation are lost, could not get them back\n");
        my $message = "No resource for reservation";
        if ($job->{message} ne $message) {
            # Sounds like it's the first time we notice that the reservation lost its resources, adding event
            OAR::IO::add_new_event($base,"UNABLE_TO_GET_RESOURCE_BACK_FOR_RESERVATION",$job->{job_id},"Resources for reservation are lost, could not get them back");
        }
        OAR::IO::set_job_message($base,$job->{job_id},$message);
    }
}
oar_debug("[MetaSched] End processing of the accepted reservations without assigned resources\n");


###########################################################################################################
# End init scheduler                                                                                      #
###########################################################################################################

# launch right reservation jobs
# arg : queue name
# return 1 if there is at least a job to treate, 2 if besteffort jobs must die
sub treate_waiting_reservation_jobs($){
    my $queue_name = shift;

    oar_debug("[MetaSched] Queue $queue_name: begin processing of waiting reservations\n");

    my $return = 0;

    my @arrayJobs = OAR::IO::get_waiting_reservation_jobs_specific_queue($base,$queue_name);
    # See if there are reserved jobs to launch
    foreach my $job (@arrayJobs){
        my $job_descriptions = OAR::IO::get_resources_data_structure_current_job($base,$job->{job_id});
        my $moldable = $job_descriptions->[0];
    
        # Test if the job is in the past
        if ($current_time_sec > $job->{start_time}+$moldable->[1] ){
            oar_warn("[MetaSched] [$job->{job_id}] set job state to Error: reservation expired and couldn't be started\n");
            OAR::IO::set_job_state($base, $job->{job_id}, "Error");
            OAR::IO::set_job_message($base,$job->{job_id},"Reservation expired and couldn't be started.");
            $return = 1;
        }
        my @resa_alive_resources;
        my $jobtypes = OAR::IO::get_job_types_hash($base,$job->{job_id});
        my $standbystart = ((defined($jobtypes->{deploy}) and $jobtypes->{deploy} eq "standby")
            or (defined($jobtypes->{cosystem}) and $jobtypes->{cosystem} eq "standby")
            or (defined($jobtypes->{noop}) and $jobtypes->{noop} eq "standby"));
        if ($standbystart){
            oar_debug("[MetaSched] [$job->{job_id}] reservation is allowed to start with some resources in standby\n");
            @resa_alive_resources = OAR::IO::get_gantt_Alive_or_Standby_resources_for_job($base,$moldable->[2], $job->{start_time} + $moldable->[1] + $Security_time_overhead);
        } else {
            @resa_alive_resources = OAR::IO::get_gantt_Alive_resources_for_job($base,$moldable->[2]);
        }
        # test if the job is going to be launched and there is no Alive node
        if (($#resa_alive_resources < 0) && ($job->{start_time} <= $current_time_sec)){
            oar_warn("[MetaSched] [$job->{job_id}] reservation is waiting because no resource is available\n");
            OAR::IO::set_gantt_job_startTime($base,$moldable->[2],$current_time_sec + 1);
        }elsif($job->{start_time} <= $current_time_sec){
            my @resa_resources = OAR::IO::get_gantt_resources_for_job($base,$moldable->[2]);
            if ($job->{start_time} + $Reservation_waiting_timeout > $current_time_sec){
                if ($#resa_resources > $#resa_alive_resources){
                    # we have not the same number of nodes than in the query --> wait the specified timeout
                    oar_warn("[MetaSched] [$job->{job_id}] reservation is waiting because not all resources are available yet (timeout in ".($job->{start_time} + $Reservation_waiting_timeout - $current_time_sec)." seconds)\n");
                    OAR::IO::set_gantt_job_startTime($base,$moldable->[2],($current_time_sec + 1));
                }
            }else{
                #Check if resources are in Alive state otherwise remove them, the job is going to be launched
                foreach my $r (@resa_resources){
                    my $resource_info = OAR::IO::get_resource_info($base,$r);
                    if (not ($resource_info->{state} eq "Alive" or ($standbystart and $resource_info->{state} eq "Absent" and
                        ($job->{start_time} + $moldable->[1] + $Security_time_overhead) < $resource_info->{available_upto}))){
                        oar_warn("[MetaSched] [$job->{job_id}] drop resource $r of the reservation, because resource is $resource_info->{state})\n");
                        OAR::IO::remove_gantt_resource_job($base, $moldable->[2], $r);
                    }
                }
                if ($#resa_resources > $#resa_alive_resources){
                    OAR::IO::add_new_event($base,"SCHEDULER_REDUCE_NB_NODES_FOR_RESERVATION",$job->{job_id},"[MetaSched] Reduce the number of resources for the job $job->{job_id}.");
                    my $n = $#resa_alive_resources + 1;
                    if ($job->{message} =~ s/R\=\d+/R\=$n/g) {
                        OAR::IO::set_job_message($base,$job->{job_id},$job->{message});
                    }
                }
            }
        }
    }
    oar_debug("[MetaSched] Queue $queue_name: end processing of waiting reservations\n");
    return($return);
}


# check for jobs with reservation
# arg : queue name, order_part
# return 1 if there is at least a job to treate else 0
sub check_reservation_jobs($$){
    my $queue_name = shift;
    my $order_part = shift;

    oar_debug("[MetaSched] Queue $queue_name: begin processing of new reservations\n");

    my $return = 0;

    #Init the gantt chart with all resources
    my $Gantt = {};
    $Gantt->{0}->{""}->{""}->{""} = OAR::Schedulers::GanttHoleStorage::new($Max_resources, $Minimum_hole_time);
    OAR::Schedulers::GanttHoleStorage::add_new_resources($Gantt->{0}->{""}->{""}->{""}, $All_resource_list_vec);

    # Find jobs to check
    my @jobs_to_sched = OAR::IO::get_waiting_toSchedule_reservation_jobs_specific_queue($base,$queue_name);
    if ($#jobs_to_sched >= 0){
        oar_debug("[MetaSched] Building the gantt structure and filling in already scheduled jobs\n");
        # Build gantt diagram of other jobs
        # Take care of currently scheduled jobs except besteffort jobs if queue_name is not besteffort
        my ($order, %already_scheduled_jobs) = OAR::IO::get_gantt_scheduled_jobs($base);
        foreach my $i (keys(%already_scheduled_jobs)){
            my $container_id = 0;
            my $inner_id = 0;
            my $placeholder_name = "";
            my $allowed_name = "";
            my $timesharing_user = "";
            my $timesharing_name = "";
            my $types = OAR::IO::get_job_types_hash($base,$i);
            if ((! defined($types->{besteffort})) or ($queue_name eq "besteffort")){
                my @resource_list = @{$already_scheduled_jobs{$i}->[3]};
                my $resource_list_vec = $already_scheduled_jobs{$i}->[10];
                my $job_duration = $already_scheduled_jobs{$i}->[1];
                if ($already_scheduled_jobs{$i}->[4] eq "Suspended"){
                    # Remove resources of the type specified in SCHEDULER_AVAILABLE_SUSPENDED_RESOURCE_TYPE
                    ($resource_list_vec, @resource_list) = OAR::IO::get_job_current_resources($base, $already_scheduled_jobs{$i}->[7],\@Sched_available_suspended_resource_type);
                    next if ($#resource_list < 0);
                }
                if ($already_scheduled_jobs{$i}->[8] eq "YES"){
                    # This job was suspended so we must recalculate the walltime
                    $job_duration += OAR::IO::get_job_suspended_sum_duration($base,$i,$current_time_sec);
                }

                if (defined($types->{inner}) and ($types->{inner} =~ /^(\d+)$/)){
                    $inner_id = $1;
                    if (defined($Gantt->{$inner_id}->{""}->{""}->{""})){
                        oar_debug("[MetaSched] [$i] inner job: using container $inner_id\n");
                    }else{
                        oar_debug("[MetaSched] [$i] inner job: container $inner_id does not exist but job is running, use container 0.\n");
                        $inner_id = 0;
                    }
                }
                if (defined($types->{container})){ # A container job cannot be placeholder or allowed or timesharing. 
                    oar_debug("[MetaSched] [$i] Job is ($inner_id,,,)\n");
                    $container_id = $i;
                    oar_debug("[MetaSched] [$i] container job: create gantt ($container_id,,,)\n");
                    $Gantt->{$container_id}->{""}->{""}->{""} = 
                        OAR::Schedulers::GanttHoleStorage::new_with_1_hole($Max_resources, $Minimum_hole_time, $already_scheduled_jobs{$i}->[0], $job_duration + $Security_time_overhead, $resource_list_vec, $All_resource_list_vec);
                } else {
                    ($placeholder_name, $allowed_name, $timesharing_user, $timesharing_name) = 
                        OAR::Schedulers::GanttHoleStorage::manage_gantt_for_timesharing_and_placeholder($Gantt, $already_scheduled_jobs{$i}->[5], $already_scheduled_jobs{$i}->[6], $types, $inner_id, "[MetaSched] [$i]");
                }

                #Fill all other gantts
                OAR::Schedulers::GanttHoleStorage::fill_gantts($Gantt, $already_scheduled_jobs{$i}->[0], $job_duration + $Security_time_overhead, $resource_list_vec, $inner_id ,$placeholder_name, $allowed_name, $timesharing_user, $timesharing_name, "[MetaSched] [$i]"); 
            }
        }
        oar_debug("[MetaSched] Try and schedule new reservations of queue $queue_name\n");
        foreach my $job (@jobs_to_sched){
            my $container_id = 0;
            my $inner_id = 0;
            my $placeholder_name = "";
            my $allowed_name = "";
            my $timesharing_user = "";
            my $timesharing_name = "";
            my $i = $job->{job_id};
            my $types = OAR::IO::get_job_types_hash($base,$i);
            my $job_descriptions = OAR::IO::get_resources_data_structure_current_job($base,$job->{job_id});
            # It is a reservation, we take care only of the first moldable job
            my $moldable = $job_descriptions->[0];
            my $duration = $moldable->[1];

            #look if reservation is too old
            if ($current_time_sec >= ($job->{start_time} + $duration)){
                oar_warn("[MetaSched] [$i] Canceling job: reservation is too old\n");
                OAR::IO::set_job_message($base, $job->{job_id}, "Reservation too old");
                OAR::IO::set_job_state($base, $job->{job_id}, "toError");
            }else{
                if ($job->{start_time} < $current_time_sec){
                    $job->{start_time} = $current_time_sec;
                    #OAR::IO::set_running_date_arbitrary($base,$job->{job_id},$current_time_sql);
                }
                
                # Get the list of resources where the reservation will be able to be launched = resources in state Alive, Absent or Suspected
                # It is assumed here that Absent and Suspected resources could become Alive by the time of the start of the advance reservation job
                my $available_resources_vector = $Alive_resource_id_vec | $Absent_resource_id_vec | $Suspected_resource_id_vec;
                if (defined($types->{inner}) and ($types->{inner} =~ /^(\d+)$/)){
                    $inner_id = $1;
                    if (defined($Gantt->{$inner_id}->{""}->{""}->{""})){
                        oar_debug("[MetaSched] [$i] inner job: using container $inner_id\n");
                    }else{
                        oar_debug("[MetaSched] [$i] inner job: container $inner_id does not exist.\n");
                        OAR::IO::set_job_message($base,$i,"Container $inner_id does not exist");
                        OAR::IO::set_job_scheduler_info($base,$i,"Container $inner_id does not exist");
                        $inner_id=0;
                        next;
                    }
                }
                if (defined($types->{container})){ # A container job cannot be placeholder or allowed or timesharing. 
                    oar_debug("[MetaSched] [$i] Job is ($inner_id,,,) and is a container (ignoring any timesharing or placeholder)\n");
                } else {
                    ($placeholder_name, $allowed_name, $timesharing_user, $timesharing_name) =
                        OAR::Schedulers::GanttHoleStorage::manage_gantt_for_timesharing_and_placeholder($Gantt, $job->{job_user}, $job->{job_name}, $types, $inner_id, "[MetaSched] [$i]");
                }

                my $job_properties = "\'1\'";
                if ((defined($job->{properties})) and ($job->{properties} ne "")){
                    $job_properties = $job->{properties};
                }

                #my $resource_id_used_list_vector = '';
                my @tree_list;
                foreach my $m (@{$moldable->[0]}){
                    my $tmp_properties = "\'1\'";
                    if ((defined($m->{property})) and ($m->{property} ne "")){
                        $tmp_properties = $m->{property};
                    }
                    my $tmp_tree = OAR::IO::get_possible_wanted_resources($base_ro,$available_resources_vector,undef,\@Dead_resource_ids,"$job_properties AND $tmp_properties", $m->{resources}, $order_part);
                    $tmp_tree = OAR::Schedulers::ResourceTree::delete_tree_nodes_with_not_enough_resources($tmp_tree);
                    push(@tree_list, $tmp_tree);
                }

                oar_debug("[MetaSched] [$i] Find first hole with gantt($inner_id,$allowed_name,$timesharing_user,$timesharing_name)\n");
                my @hole = OAR::Schedulers::GanttHoleStorage::find_first_hole($Gantt->{$inner_id}->{$allowed_name}->{$timesharing_user}->{$timesharing_name},$job->{start_time}, $duration + $Security_time_overhead, \@tree_list, 30);
                if ($hole[0] == $job->{start_time}){
                    # The reservation can be scheduled
                    my @resources;
                    foreach my $t (@{$hole[1]}){
                        foreach my $r (OAR::Schedulers::ResourceTree::get_tree_leafs($t)){
                            push(@resources, OAR::Schedulers::ResourceTree::get_current_resource_value($r));
                        }
                    }
            
                    # We can schedule the job
                    oar_warn("[MetaSched] [$i] advance reservation is validated\n");
                    my $vec = '';
                    foreach my $r (@resources){
                        vec($vec, $r, 1) = 1;
                    }
                    # Create gantt for the new container
                    if (defined($types->{container})){
                        $container_id = $i;
                        oar_debug("[MetaSched] [$i] container job: create gantt ($container_id,,,)\n");
                        $Gantt->{$container_id}->{""}->{""}->{""} = OAR::Schedulers::GanttHoleStorage::new_with_1_hole($Max_resources, $Minimum_hole_time, $job->{start_time}, $duration  + $Security_time_overhead, $vec, $All_resource_list_vec);
                    }
        
                    #Fill all other gantts
                    OAR::Schedulers::GanttHoleStorage::fill_gantts($Gantt, $job->{start_time}, $duration + $Security_time_overhead, $vec, $inner_id, $placeholder_name, $allowed_name, $timesharing_user, $timesharing_name, "[MetaSched] [$i]"); 

                    # Update database
                    push(@resources, @Resources_to_always_add);
                    oar_debug("[MetaSched] [$i] add job in database\n");
                    OAR::IO::add_gantt_scheduled_jobs($base,$moldable->[2],$job->{start_time},\@resources);
                    OAR::IO::set_job_state($base, $job->{job_id}, "toAckReservation");
                }else{           
                    oar_warn("[MetaSched] [$i] advance reservation cannot be validated, not enough resources\n");
                    OAR::IO::set_job_state($base, $job->{job_id}, "toError");
                    if ($hole[0] == OAR::Schedulers::GanttHoleStorage::get_infinity_value()){
                        OAR::IO::set_job_message($base, $job->{job_id}, "This reservation cannot run");
                    }else{
                        OAR::IO::set_job_message($base, $job->{job_id}, "This reservation could run at ".OAR::IO::local_to_sql($hole[0]));
                    }
                }
            }
            OAR::IO::set_job_resa_state($base, $job->{job_id}, "Scheduled");
            $return = 1;
        }
    }
    oar_debug("[MetaSched] Queue $queue_name: end processing of new reservations\n");
    return($return);
}


# Detect if there are besteffort jobs to kill
# return 1 if there is at least 1 job to frag otherwise 0
sub check_jobs_to_kill(){
    oar_debug("[MetaSched] Begin precessing of besteffort jobs to kill\n");
    my $return = 0;
    my %nodes_for_jobs_to_launch = OAR::IO::get_gantt_resources_for_jobs_to_launch($base,$current_time_sec);
    my %fragged_jobs = ();
    foreach my $r (keys(%nodes_for_jobs_to_launch)){
        if (defined($besteffort_resource_occupation{$r})) {
            if (OAR::IO::is_timesharing_for_2_jobs($base, $nodes_for_jobs_to_launch{$r}, $besteffort_resource_occupation{$r}->{job_id})) {
                oar_debug("[MetaSched] Resource $r is needed for job $nodes_for_jobs_to_launch{$r}, but besteffort job $besteffort_resource_occupation{$r}->{job_id} can live, because timesharing compatible\n");
            } else {
                unless (defined($fragged_jobs{$besteffort_resource_occupation{$r}->{job_id}})) {
                    my $skip_kill = 0;
                    # Check if we must checkpoint the besteffort job
                    if ($besteffort_resource_occupation{$r}->{checkpoint} > 0){
                        my $checkpoint_first_date;
                        foreach my $e (OAR::IO::get_job_events($base, $besteffort_resource_occupation{$r}->{job_id})){
                            if ($e->{type} eq "CHECKPOINT"){
                                if ((!defined($checkpoint_first_date)) or ($checkpoint_first_date > $e->{date})){
                                    $checkpoint_first_date = $e->{date};
                                }
                            }
                        }
                        if ((!defined($checkpoint_first_date)) or ($current_time_sec <= ($checkpoint_first_date+$besteffort_resource_occupation{$r}->{checkpoint}))){
                            # Retrieve node names used by the job
                            my @hosts = OAR::IO::get_job_current_hostnames($base,$besteffort_resource_occupation{$r}->{job_id});
                            my $types = OAR::IO::get_job_types_hash($base,$besteffort_resource_occupation{$r}->{job_id});
                            if ($#hosts < 0 and not defined($types->{cosystem}) and not defined($types->{deploy})) {
                                oar_debug("[MetaSched] Not sending checkpoint signal to the job $besteffort_resource_occupation{$r}->{job_id} (job with no host)\n");
                            } else {
                                my $host_to_connect = $hosts[0];
                                if (defined($types->{cosystem})) {
                                    $host_to_connect = $Cosystem_hostname;
                                }elsif (defined($types->{deploy})){
                                    $host_to_connect = $Deploy_hostname;
                                }
                                oar_debug("[MetaSched] Sending checkpoint signal to the job $besteffort_resource_occupation{$r}->{job_id} on host $host_to_connect\n");
                                OAR::IO::add_new_event($base,"CHECKPOINT",$besteffort_resource_occupation{$r}->{job_id},"User oar (metasched) requested a checkpoint on the job $besteffort_resource_occupation{$r}->{job_id} on $host_to_connect");
                                $skip_kill = 1;
                                my $str_comment;
                                my @exit_codes;
                                # Timeout the ssh command
                                eval {
                                    $SIG{ALRM} = sub { die "alarm\n" };
                                    alarm(OAR::Tools::get_ssh_timeout());
                                    @exit_codes = OAR::Tools::signal_oarexec($host_to_connect,$besteffort_resource_occupation{$r}->{job_id},"SIGUSR2",1,$base, $Openssh_cmd, '');
                                    alarm(0);
                                };
                                if ($@){
                                    if ($@ eq "alarm\n"){
                                        $str_comment = "[MetaSched] Cannot contact $host_to_connect, operation timouted (".OAR::Tools::get_ssh_timeout()." s). So I cannot send checkpoint signal to the job $besteffort_resource_occupation{$r}->{job_id}";
                                        oar_warn("$str_comment\n");
                                        OAR::IO::add_new_event($base,"CHECKPOINT_ERROR",$besteffort_resource_occupation{$r}->{job_id},$str_comment);
                                    }else{
                                        $str_comment = "[MetaSched] An unknown error occured during the sending of the checkpoint signal to the job $besteffort_resource_occupation{$r}->{job_id} on the host $host_to_connect";
                                        oar_warn("$str_comment\n");
                                        OAR::IO::add_new_event($base,"CHECKPOINT_ERROR",$besteffort_resource_occupation{$r}->{job_id},$str_comment);
                                    }
                                }else{
                                    if ($exit_codes[0] == 0){
                                        $str_comment = "[MetaSched] The job $besteffort_resource_occupation{$r}->{job_id} was notified to checkpoint itself on the node $host_to_connect";
                                        oar_debug("$str_comment\n");
                                        OAR::IO::add_new_event($base,"CHECKPOINT_SUCCESSFULL",$besteffort_resource_occupation{$r}->{job_id},$str_comment);
                                    }else{
                                        $str_comment = "[MetaSched] The kill command return a bad exit code (@exit_codes) for the job $besteffort_resource_occupation{$r}->{job_id} on the node $host_to_connect";
                                        oar_warn("$str_comment\n");
                                        OAR::IO::add_new_event($base,"CHECKPOINT_ERROR",$besteffort_resource_occupation{$r}->{job_id},$str_comment);
                                    }
                                }
                            }
                        }else{
                            # Retrieve node names used by the job
                            my @hosts = OAR::IO::get_job_current_hostnames($base,$besteffort_resource_occupation{$r}->{job_id});
                            my $types = OAR::IO::get_job_types_hash($base,$besteffort_resource_occupation{$r}->{job_id});
                            if ($#hosts >= 0 or defined($types->{cosystem}) or defined($types->{deploy})) {
                                my $str_comment = "[MetaSched] Checkpoint timeout for job $besteffort_resource_occupation{$r}->{job_id} ($current_time_sec > ($checkpoint_first_date+$besteffort_resource_occupation{$r}->{checkpoint})";
                                oar_debug("$str_comment\n");
                                OAR::IO::add_new_event($base,"CHECKPOINT_TIMEOUT",$besteffort_resource_occupation{$r}->{job_id},$str_comment);
                            }
                        }
                    }
                    if ($skip_kill == 0){
                        oar_debug("[MetaSched] Resource $r need to be freed for job $nodes_for_jobs_to_launch{$r}: killing besteffort job $besteffort_resource_occupation{$r}->{job_id}\n");
                        OAR::IO::add_new_event($base,"BESTEFFORT_KILL",$besteffort_resource_occupation{$r}->{job_id},"[MetaSched] kill the besteffort job $besteffort_resource_occupation{$r}->{job_id}");
                        OAR::IO::lock_table($base,["frag_jobs","event_logs","jobs"]);
                        OAR::IO::frag_job($base, $besteffort_resource_occupation{$r}->{job_id});
                        OAR::IO::unlock_table($base);
                    }
                    $fragged_jobs{$besteffort_resource_occupation{$r}->{job_id}} = 1;
                    $return = 1;
                }
            }
        }
    }
    oar_debug("[MetaSched] End precessing of besteffort jobs to kill\n");
    return($return);
}

# Tell Almighty to run a job
sub notify_to_run_job($$){
    my ($base,
        $job_id) = @_;

    if (!defined($To_launch_jobs_already_treated{$job_id})){
        if (OAR::IO::is_job_desktop_computing($base, $job_id)){
            oar_debug("[MetaSched] $job_id : Desktop computing job, I don't handle it!\n");
        }else{
            my $err = OAR::Tools::notify_tcp_socket($Remote_host,$Remote_port,"OARRUNJOB_$job_id");
            if (!defined($err)){
                $To_launch_jobs_already_treated{$job_id} = 1;
                oar_debug("[MetaSched] Notify almighty to launch the job $job_id\n");
            }else{
                oar_warn("[MetaSched] Not able to notify almighty to launch the job $job_id ($err)\n");
            }
        }
    }
}

# Prepare a job to be run by bipbip
sub prepare_job_to_be_launched($$$$$){
    my ($base,
        $job_id,
        $moldable_job_id,
        $job_submission_time,
        $resources_array_ref) = @_;

    my $running_date = $current_time_sec;
    if ($running_date < $job_submission_time){
        $running_date = $job_submission_time;
    }
    OAR::IO::set_running_date_arbitrary($base, $job_id, $running_date);
    OAR::IO::set_assigned_moldable_job($base, $job_id, $moldable_job_id);

    my $insert_from_file = get_conf_with_default_param("INSERTS_FROM_FILE", "no");
    if ($insert_from_file eq 'yes') {
        OAR::IO::add_resource_job_pairs_from_file($base, $moldable_job_id, $resources_array_ref);
    }else{
        OAR::IO::add_resource_job_pairs($base, $moldable_job_id, $resources_array_ref);
    }
    OAR::IO::set_job_state($base, $job_id, "toLaunch");
    notify_to_run_job($base, $job_id);
}


# Detect if there are jobs to launch
# return 1 if there is at least 1 job to launch otherwise 0
sub check_jobs_to_launch(){
    oar_debug("[MetaSched] Begin processing of jobs to launch (start time <= $current_time_sql)\n");
    my $return_code = 0;
    my %jobs_to_launch = OAR::IO::get_gantt_jobs_to_launch($base,$current_time_sec);
    
    foreach my $i (keys(%jobs_to_launch)){
        oar_debug("[MetaSched] Set job $i state to toLaunch at $current_time_sql\n");
        # We must look at reservations to not go after the initial stop time
        my $mold = OAR::IO::get_current_moldable_job($base,$jobs_to_launch{$i}->[0]);
        my $job = OAR::IO::get_job($base,$i);
        if (($job->{reservation} eq "Scheduled") and ($job->{start_time} < $current_time_sec)){
            my $max_time = $mold->{moldable_walltime} - ($current_time_sec - $job->{start_time});
            OAR::IO::set_moldable_job_max_time($base,$jobs_to_launch{$i}->[0], $max_time);
            OAR::IO::set_gantt_job_startTime($base,$jobs_to_launch{$i}->[0],$current_time_sec);
            oar_warn("[MetaSched] Reduce walltime of job $i to $max_time (was  $mold->{moldable_walltime})\n");
            OAR::IO::add_new_event($base,"REDUCE_RESERVATION_WALLTIME",$i,"Change walltime from $mold->{moldable_walltime} to $max_time");
            my $w = OAR::IO::duration_to_sql($max_time);                                                                            
            if ($job->{message} =~ s/W\=\d+\:\d+\:\d+/W\=$w/g){                                                                     
                OAR::IO::set_job_message($base,$i,$job->{message});                                                                  
            }
        }
        prepare_job_to_be_launched($base, $i, $jobs_to_launch{$i}->[0], $job->{submission_time}, $jobs_to_launch{$i}->[1]);
        $return_code = 1;
    }

    oar_debug("[MetaSched] End processing of jobs to launch\n");
    return($return_code);
}

my %initial_time = (
                "sec" => $current_time_sec,
                "sql" => $current_time_sql
              );

my @queues = OAR::IO::get_active_queues($base);
my $name;
my $policy;
foreach my $i (@queues){
    $name = $i->[0];
    $policy = $i->[1];
    my $waiting_jobs = OAR::IO::is_waiting_job_specific_queue_present($base,$name);
    if ($waiting_jobs == 1){
        oar_debug("[MetaSched] Queue $name: Launching scheduler $policy at time $initial_time{sql}\n");
        my ($sched_exit_code,$sched_signal_num,$sched_dumped_core) = (-1, -1, -1);
        my $sched_pid = open(READSCHED, "$binpath/schedulers/$policy $name $initial_time{sec} \"$initial_time{sql}\" |") or oar_warn("[MetaSched] Cannot run: \"$binpath/schedulers/$policy $name $initial_time{sec} \"$initial_time{sql}\"\"; $!\n");
        if (defined($sched_pid)){
            while(my $l = <READSCHED>){
                chop($l);
                oar_debug("[MetaSched] Read on the scheduler output:$l\n");
                if ((get_conf_with_default_param("SCHEDULER_LAUNCHER_OPTIMIZATION", "yes") eq "yes") and
                    ($l =~ m/^SCHEDRUN JOB_ID=(\d+) MOLDABLE_JOB_ID=(\d+) RESOURCES=([\d,]+)$/m)){
                    my $job_id_to_schedule = $1;
                    my $moldable_job_id_to_schedule = $2;
                    my @job_resources_to_schedule = split(',',$3);
                    my $skip = "";
                    foreach my $r (@job_resources_to_schedule){
                        if (defined($besteffort_resource_occupation{$r})){
                            $skip = "a besteffort job needs to be killed first";
                            last;
                        }elsif(vec($Absent_resource_id_vec, $r, 1)){
                            $skip = "some nodes need to be booted first";
                        }
                    }
                    if ($skip eq ""){
                        oar_debug("[MetaSched] Early launch the job $job_id_to_schedule\n");
                        my $job = OAR::IO::get_job($base,$job_id_to_schedule);
                        prepare_job_to_be_launched($base, $job_id_to_schedule, $moldable_job_id_to_schedule, $job->{submission_time}, \@job_resources_to_schedule);
                    }else{
                        oar_debug("[MetaSched] Not able to early launch $job_id_to_schedule: $skip\n");
                    }
                }
            }
            close(READSCHED);
            $sched_exit_code  = $? >> 8;
            $sched_signal_num  = $? & 127;
            $sched_dumped_core = $? & 128;
        }
        if ((!defined($sched_pid)) or ($sched_signal_num != 0) or ($sched_dumped_core != 0)){
            oar_error("[MetaSched] Error: execution of $policy failed (pid=".(defined($sched_pid)?"undef":$sched_pid)." signal=$sched_signal_num core_dump=$sched_dumped_core). Disabling queue $name (see `oarnotify')\n");
            OAR::IO::stop_a_queue($base,$name);
        }
        if ($sched_exit_code == 1){
            $exit_code = 0;
        }elsif($sched_exit_code != 0){
            oar_error("[MetaSched] Error: Scheduler $policy returned a bad value: $sched_exit_code, at time $initial_time{sec}. Disabling queue $name (see `oarnotify')\n");
            OAR::IO::stop_a_queue($base,$name);
        }
        treate_waiting_reservation_jobs($name);
        check_reservation_jobs($name,$Order_part);
    }else{
        oar_debug("[MetaSched] Queue $name: No job\n");
    }
}

if ($exit_code == 0){
    if (check_jobs_to_kill() == 1){
        # We must kill besteffort jobs
        OAR::Tools::notify_tcp_socket($Remote_host,$Remote_port,"ChState");
        $exit_code = 2;
    }elsif (check_jobs_to_launch() == 1){
        $exit_code = 0;
    }
}

#Update visu gantt tables
OAR::IO::update_gantt_visualization($base); 
OAR::IO::disconnect($base_ro);

# Manage dynamic node feature
my $timeout_cmd = 10;
my $flagHulot=0;
if (is_conf("SCHEDULER_TIMEOUT")){
    $timeout_cmd = get_conf("SCHEDULER_TIMEOUT");
}
if ((is_conf("SCHEDULER_NODE_MANAGER_SLEEP_CMD") 
          or (get_conf("ENERGY_SAVING_INTERNAL") eq "yes" and is_conf("ENERGY_SAVING_NODE_MANAGER_SLEEP_CMD")))
          and is_conf("SCHEDULER_NODE_MANAGER_SLEEP_TIME") and is_conf("SCHEDULER_NODE_MANAGER_IDLE_TIME")){

    # Look at nodes that are unused for a duration
    my $idle_duration = get_conf("SCHEDULER_NODE_MANAGER_IDLE_TIME");
    my $sleep_duration = get_conf("SCHEDULER_NODE_MANAGER_SLEEP_TIME");

    my %nodes = OAR::IO::search_idle_nodes($base, $current_time_sec);
    my $tmp_time = $current_time_sec - $idle_duration;
    my @node_halt;
    foreach my $n (keys(%nodes)){
        if ($nodes{$n} < $tmp_time){
            # Search if the node has enough time to sleep
            my $tmp = OAR::IO::get_next_job_date_on_node($base,$n);
            if (!defined($tmp) or ($tmp - $sleep_duration > $current_time_sec)){
                # Search if node has not been woken up recently
                my $wakeup_date = OAR::IO::get_last_wake_up_date_of_node($base,$n);
                if (!defined($wakeup_date) or ($wakeup_date < $tmp_time)){
                    push(@node_halt, $n);
                }
            }
        }
    }

    if ($#node_halt >= 0){
        oar_debug("[MetaSched] Powering off some nodes (energy saving): @node_halt\n");
        # Using the built-in energy saving module to shut down nodes
        if (get_conf_with_default_param("ENERGY_SAVING_INTERNAL", "no") eq "yes") {
            if (OAR::Modules::Hulot::halt_nodes(\@node_halt) ) {
                oar_error("[MetaSched] Error: communication problem with the energy saving module (Hulot)\n");
            }
            $flagHulot=1;
        }
        # Not using the built-in energy saving module to shut down nodes
        else {
            my $cmd = get_conf("SCHEDULER_NODE_MANAGER_SLEEP_CMD");
            if (! defined(OAR::Tools::fork_and_feed_stdin($cmd, $timeout_cmd, \@node_halt))){
                oar_error("[MetaSched] Error: command $cmd timeouted (${timeout_cmd}s) while trying to poweroff some nodes\n");
            }
        }
    }
}

if (is_conf("SCHEDULER_NODE_MANAGER_WAKE_UP_CMD") or (get_conf("ENERGY_SAVING_INTERNAL") eq "yes" and is_conf("ENERGY_SAVING_NODE_MANAGER_WAKE_UP_CMD"))){
    # Get nodes which the scheduler wants to schedule jobs to, but which are in the Absent state, to wake them up.
    my $wakeup_time = get_conf_with_default_param("SCHEDULER_NODE_MANAGER_WAKEUP_TIME", 1);
    my @nodes = OAR::IO::get_gantt_hostname_to_wake_up($base, $current_time_sec, $wakeup_time);

    if ($#nodes >= 0){
        oar_debug("[MetaSched] Waking-up some nodes: @nodes\n");
        # Using the built-in energy saving module to wake up nodes
        if (get_conf_with_default_param("ENERGY_SAVING_INTERNAL", "no") eq "yes") {
            if (OAR::Modules::Hulot::wake_up_nodes(\@nodes) ) {
                oar_error("[MetaSched] Error: Communication problem with the energy saving module (Hulot)\n");
            }
						$flagHulot=1;
        }
        # Not using the built-in energy saving module to wake up nodes
        else {
            my $cmd = get_conf("SCHEDULER_NODE_MANAGER_WAKE_UP_CMD");
            if (! defined(OAR::Tools::fork_and_feed_stdin($cmd, $timeout_cmd, \@nodes))){
                oar_error("[MetaSched] Error: command $cmd timeouted (${timeout_cmd}s) while trying to wake-up some nodes\n");
            }
        }
    } else {
        oar_debug("[MetaSched] No node to wake-up\n");
    }
}

# Send CHECK signal to Hulot if needed
if(!$flagHulot and (get_conf_with_default_param("ENERGY_SAVING_INTERNAL", "no") eq "yes")){
    if (OAR::Modules::Hulot::check() ) {
        oar_error("[MetaSched] Error: communication problem with the energy saving module (Hulot)\n");
    }
}

# Search jobs to resume
foreach my $j (OAR::IO::get_jobs_in_state($base, "Resuming")){
    my @other_jobs = OAR::IO::get_jobs_on_resuming_job_resources($base,$j->{job_id});
    # TODO : look for timesharing other jobs. What do we do?????
    if ($#other_jobs < 0){
        # We can resume the job
        oar_debug("[MetaSched] [$j->{job_id}] Resuming job\n");
        my $jobtypes = OAR::IO::get_job_types_hash($base,$j->{job_id});
        if (defined($jobtypes->{noop})){
            OAR::IO::resume_job_action($base,$j->{job_id});
            oar_debug("[MetaSched] [$j->{job_id}] Resume NOOP job OK\n");
        }else{
            ###############
            # RESUME PART #
            ###############
            my $script = get_conf("JUST_BEFORE_RESUME_EXEC_FILE");
            my $timeout = get_conf("SUSPEND_RESUME_SCRIPT_TIMEOUT");
            $timeout = OAR::Tools::get_default_suspend_resume_script_timeout() if (!defined($timeout));
            my $skip = 0;
            if (defined($script)){
                # Launch admin script
                my $script_error = 0;
                eval {
                    $SIG{ALRM} = sub { die "alarm\n" };
                    alarm($timeout);
                    oar_debug("[MetaSched] [$j->{job_id}] Running post suspend script: `$script $j->{job_id}'\n");
                    $script_error = system("$script script $j->{job_id}");
                    oar_debug("[MetaSched] [$j->{job_id}] `$script $j->{job_id}' terminated\n");
                    alarm(0);
                };
                if( $@ || ($script_error != 0)){
                    my $str = "[MetaSched] [$j->{job_id}] Suspend script error: $@; return code = $script_error\n";
                    oar_error($str);
                    OAR::IO::add_new_event($base,"RESUME_SCRIPT_ERROR",$j->{job_id},$str);
                    OAR::IO::frag_job($base,$j->{job_id});
                    OAR::Tools::notify_tcp_socket($Remote_host,$Remote_port,"Qdel");
                    $skip = 1;
                }
            }

            if ((defined($Cpuset_field)) and ($skip == 0)){
                my $cpuset_name = OAR::IO::get_job_cpuset_name($base, $j->{job_id}) if (defined($Cpuset_field));
                my $cpuset_nodes = OAR::IO::get_cpuset_values_for_a_moldable_job($base,$Cpuset_field,$j->{assigned_moldable_job});
                my $suspend_data_hash = {
                    name => $cpuset_name,
                    job_id => $j->{job_id},
                    oarexec_pid_file => OAR::Tools::get_oar_pid_file_name($j->{job_id}),
                };
                if (defined($cpuset_nodes)){
                    my $taktuk_cmd = get_conf("TAKTUK_CMD");
                    my $suspend_file = get_conf("SUSPEND_RESUME_FILE");
                    $suspend_file = OAR::Tools::get_default_suspend_resume_file() if (!defined($suspend_file));
                    $suspend_file = "$ENV{OARDIR}/$suspend_file" if ($suspend_file !~ /^\//);
                    my ($tag,@bad) = OAR::Tools::manage_remote_commands([keys(%{$cpuset_nodes})],$suspend_data_hash,$suspend_file,"resume",$Openssh_cmd,$taktuk_cmd,$base);
                    if ($tag == 0){
                        my $str = "[MetaSched] [SUSPEND_RESUME] [$j->{job_id}] Bad suspend/resume file: $suspend_file\n";
                        oar_error($str);
                        OAR::IO::add_new_event($base, "SUSPEND_RESUME_MANAGER_FILE", $j->{job_id}, $str);
                    }else{
                       if (($#bad < 0)){
                            OAR::IO::resume_job_action($base,$j->{job_id});
                       }else{
                            my $str = "[MetaSched] [SUSPEND_RESUME] [$j->{job_id}] Error on several nodes: @bad\n";
                            oar_error($str);
                            OAR::IO::add_new_event_with_host($base,"RESUME_ERROR",$j->{job_id},$str,\@bad);
                            OAR::IO::frag_job($base,$j->{job_id});
                            # A Leon must be run
                            $exit_code = 2;
                        }
                    }
                }
            }
            #####################
            # RESUME PART, END  #
            #####################
        }
    }
}

# Notify oarsub -I when they will be launched
foreach my $j (OAR::IO::get_gantt_waiting_interactive_prediction_date($base)){
    my ($addr,$port) = split(/:/,$j->{info_type});
    my $new_start_prediction = OAR::IO::local_to_sql($j->{start_time});
    oar_debug("[MetaSched] [$j->{job_id}] Notifying user of the start prediction: $new_start_prediction (".$j->{message}.")\n");
    OAR::Tools::notify_tcp_socket($addr,$port,"[$initial_time{sql}] Start prediction: $new_start_prediction (".$j->{message}.")");
}

# Run the decisions
## Treate "toError" jobs
foreach my $j (OAR::IO::get_jobs_in_state($base,"toError")){
    my ($addr,$port) = split(/:/,$j->{info_type});
    oar_debug("[MetaSched] Treate job $j->{job_id} in toError state\n");
    if (($j->{job_type} eq "INTERACTIVE") or
        (($j->{job_type} eq "PASSIVE") and
         ($j->{reservation} eq "Scheduled"))){
        oar_debug("[MetaSched] Notify oarsub job (num:$j->{job_id}) in error; jobInfo=$j->{info_type}\n");
        if ((defined(OAR::Tools::notify_tcp_socket($addr,$port,"$j->{message}"))) or
            (defined(OAR::Tools::notify_tcp_socket($addr,$port,"BAD JOB")))){
            oar_debug("[MetaSched] Cannot open connection to oarsub client for job $j->{job_id} !\n");
        }
    }
    oar_debug("[MetaSched] Set job $j->{job_id} to state Error\n");
    OAR::IO::set_job_state($base, $j->{job_id}, "Error");
}
## Treate toAckReservation jobs
foreach my $j (OAR::IO::get_jobs_in_state($base,"toAckReservation")){
    my ($addr,$port) = split(/:/,$j->{info_type});
    oar_debug("[MetaSched] Treate job $j->{job_id} in toAckReservation state\n");
    my $err = OAR::Tools::notify_tcp_socket($addr,$port,"GOOD RESERVATION");
    if (defined($err)){
        oar_warn("[MetaSched] Frag job $j->{job_id}, I cannot notify oarsub for the reservation ($err)\n");
        OAR::IO::add_new_event($base,"CANNOT_NOTIFY_OARSUB",$j->{job_id},"[MetaSched] Can not notify oarsub for the job $j->{job_id} ($err)");
        OAR::IO::lock_table($base,["frag_jobs","event_logs","jobs"]);
        OAR::IO::frag_job($base,$j->{job_id});
        OAR::IO::unlock_table($base);
        $exit_code = 2;
    }else{
        oar_debug("[MetaSched] Notify oarsub for a RESERVATION (idJob=$j->{job_id}) --> OK; jobInfo=$j->{info_type}\n");
        OAR::IO::set_job_state($base,$j->{job_id} , "Waiting");
        if ($j->{start_time} - 1 <= $current_time_sec){
            $exit_code = 1 if ($exit_code == 0);
        }
    }
}
## Treate toLaunch jobs
foreach my $j (OAR::IO::get_jobs_in_state($base,"toLaunch")){
    notify_to_run_job($base, $j->{job_id});
}

OAR::IO::disconnect($base);
oar_debug("[MetaSched] End of Meta Scheduler\n");

exit($exit_code);
oar-server 2.5.6-2ubuntu1 / usr / lib / oar / oar_meta_sched
