#!/usr/bin/perl
#-d:DProf
#-d:NYTProf     --> nytprofhtml

use strict;
use DBI();
use OAR::IO;
use Data::Dumper;
use OAR::Modules::Judas qw(oar_debug oar_warn oar_info oar_error set_current_log_category);
use OAR::Conf qw(init_conf dump_conf get_conf is_conf get_conf_with_default_param);
use OAR::Schedulers::GanttHoleStorage_with_quotas;
use OAR::Schedulers::QuotaStorage;
use Storable qw(dclone);
use Time::HiRes qw(gettimeofday);

select(STDOUT);
$| = 1;

# Log category
set_current_log_category('scheduler');
my $scheduler_name = "SchedFSTSQ";
my $Quota_file     = "/etc/oar/scheduler_quotas.conf";
my $Session_id     = $$;

init_conf($ENV{OARCONFFILE});

###############################################################################
# Fairsharing parameters #
##########################
# Avoid problems if there are too many waiting jobs
my $Karma_max_number_of_jobs_treated_per_user = 30;
if (is_conf("SCHEDULER_FAIRSHARING_MAX_JOB_PER_USER")) {
    $Karma_max_number_of_jobs_treated_per_user = get_conf("SCHEDULER_FAIRSHARING_MAX_JOB_PER_USER");
}

# number of seconds to consider for the fairsharing
my $Karma_window_size = 3600 * 30 * 24;
if (is_conf("SCHEDULER_FAIRSHARING_WINDOW_SIZE")) {
    $Karma_window_size = get_conf("SCHEDULER_FAIRSHARING_WINDOW_SIZE");
}

# specify the target percentages for project names (0 if not specified)
my $Karma_project_targets = { first => 75, default => 25 };
if (is_conf("SCHEDULER_FAIRSHARING_PROJECT_TARGETS")) {
    $Karma_project_targets = eval(get_conf("SCHEDULER_FAIRSHARING_PROJECT_TARGETS") . ";");
    if ($@) {
        oar_error($scheduler_name,
            "Syntax error in configuration file: SCHEDULER_FAIRSHARING_PROJECT_TARGETS\n",
            $Session_id);
        exit(1);
    }
}

# specify the target percentages for users (0 if not specified)
my $Karma_user_targets = { oar => 100 };
if (is_conf("SCHEDULER_FAIRSHARING_USER_TARGETS")) {
    $Karma_user_targets = eval(get_conf("SCHEDULER_FAIRSHARING_USER_TARGETS") . ";");
    if ($@) {
        oar_error($scheduler_name,
            "Syntax error in configuration file: SCHEDULER_FAIRSHARING_USER_TARGETS\n",
            $Session_id);
        exit(1);
    }
}

#print Dumper($Karma_user_targets);
# weight given to each criteria
my $Karma_coeff_project_consumption = 0;
if (is_conf("SCHEDULER_FAIRSHARING_COEF_PROJECT")) {
    $Karma_coeff_project_consumption = get_conf("SCHEDULER_FAIRSHARING_COEF_PROJECT");
}
my $Karma_coeff_user_consumption = 2;
if (is_conf("SCHEDULER_FAIRSHARING_COEF_USER")) {
    $Karma_coeff_user_consumption = get_conf("SCHEDULER_FAIRSHARING_COEF_USER");
}
my $Karma_coeff_user_asked_consumption = 1;
if (is_conf("SCHEDULER_FAIRSHARING_COEF_USER_ASK")) {
    $Karma_coeff_user_asked_consumption = get_conf("SCHEDULER_FAIRSHARING_COEF_USER_ASK");
}

###############################################################################
# Tokens #
##########
my $Token_scripts = {};
if (is_conf("SCHEDULER_TOKEN_SCRIPTS")) {
    $Token_scripts = eval(get_conf("SCHEDULER_TOKEN_SCRIPTS") . ";");
    if ($@) {
        oar_error($scheduler_name, "Syntax error in configuration file: SCHEDULER_TOKEN_SCRIPTS\n",
            $Session_id);
        exit(1);
    }
}
###############################################################################
# Quotas #
##########
my ($Gantt_quotas, $err) = OAR::Schedulers::QuotaStorage::read_conf_file("$Quota_file");
if (defined($err)) {
    oar_error($scheduler_name, "$err; QUOTA DISABLED\n", $Session_id);
}
###############################################################################
my $initial_time            = time();
my $timeout                 = 10;
my $Minimum_timeout_per_job = 0;
if (is_conf("SCHEDULER_TIMEOUT")) {
    $timeout = get_conf("SCHEDULER_TIMEOUT");
}

# $security_time_overhead is the security time (second) used to be sure there
# are no problem with overlaping jobs
my $security_time_overhead = 60;
if (is_conf("SCHEDULER_JOB_SECURITY_TIME")) {
    $security_time_overhead = get_conf("SCHEDULER_JOB_SECURITY_TIME");
}

# Constant duration time of a besteffort job
my $besteffort_duration = 5 * 60;
$besteffort_duration = $security_time_overhead if ($besteffort_duration < $security_time_overhead);
my $besteffort_kill_duration =
  get_conf_with_default_param("SCHEDULER_BESTEFFORT_KILL_DURATION_BEFORE_RESERVATION", 0);
$besteffort_duration = $besteffort_kill_duration
  if ($besteffort_duration < $besteffort_kill_duration);

my $minimum_hole_time = 0;
if (is_conf("SCHEDULER_GANTT_HOLE_MINIMUM_TIME")) {
    $minimum_hole_time = get_conf("SCHEDULER_GANTT_HOLE_MINIMUM_TIME");
}

my $Order_part = get_conf("SCHEDULER_RESOURCE_ORDER");

my @Sched_available_suspended_resource_type;
my $sched_available_suspended_resource_type_tmp =
  get_conf("SCHEDULER_AVAILABLE_SUSPENDED_RESOURCE_TYPE");
if (!defined($sched_available_suspended_resource_type_tmp)) {
    push(@Sched_available_suspended_resource_type, "default");
} else {
    @Sched_available_suspended_resource_type =
      split(" ", $sched_available_suspended_resource_type_tmp);
}

# Look at resources that we must add for each job
my $Resources_to_always_add_type = get_conf("SCHEDULER_RESOURCES_ALWAYS_ASSIGNED_TYPE");
my @Resources_to_always_add      = ();

my $current_time;

my $queue;
if (defined($ARGV[0]) && defined($ARGV[1]) && $ARGV[1] =~ m/\d+/m) {
    $queue        = $ARGV[0];
    $current_time = $ARGV[1];
} else {
    oar_error($scheduler_name, "No queue specified on command line\n", $Session_id);
    exit(1);
}

# Init
my $base    = OAR::IO::connect();
my $base_ro = OAR::IO::connect_ro();

oar_info($scheduler_name, "Starting scheduler for queue $queue at time $current_time\n",
    $Session_id);

# First check states of resources that we must add for every job
if (defined($Resources_to_always_add_type)) {
    my $tmp_result_state_resources =
      OAR::IO::get_specific_resource_states($base, $Resources_to_always_add_type);
    if ($#{ $tmp_result_state_resources->{"Suspected"} } >= 0) {
        oar_warn(
            $scheduler_name,
            "Some of the resources matching the SCHEDULER_RESOURCES_ALWAYS_ASSIGNED_TYPE configuration directive are Suspected. No job can be scheduled. Exiting\n",
            $Session_id);
        exit(1);
    } else {
        if (defined($tmp_result_state_resources->{"Alive"})) {
            @Resources_to_always_add = @{ $tmp_result_state_resources->{"Alive"} };
            oar_info(
                $scheduler_name,
                "The following Alive resources matching the SCHEDULER_RESOURCES_ALWAYS_ASSIGNED_TYPE configuration directive will be added to every job: @Resources_to_always_add\n",
                $Session_id);
        }
    }
}

my $timesharing_gantts;

# Create the Gantt Diagrams
#Init the gantt chart with all resources
my ($Max_resources, $All_resource_list_vec, $Only_default_type_resource_vec) =
  OAR::IO::get_vecs_resources($base);

my %Container_gantt_hash;
$Container_gantt_hash{0} =
  OAR::Schedulers::GanttHoleStorage_with_quotas::new($Max_resources, $minimum_hole_time);
OAR::Schedulers::GanttHoleStorage_with_quotas::add_new_resources($Container_gantt_hash{0},
    $All_resource_list_vec);
my $quota_accounting_data = OAR::Schedulers::QuotaStorage::new($security_time_overhead);

sub parse_timesharing($$$) {
    my $str      = shift;
    my $job_user = shift;
    my $job_name = shift;

    my $user = "*";
    my $name = "*";
    foreach my $s (split(',', $str)) {
        if ($s =~ m/^\s*([\w\*]+)\s*$/m) {
            if ($1 eq "user") {
                $user = $job_user;
            } elsif (($1 eq "name") and ($job_name ne "")) {
                $name = $job_name;
            }
        }
    }

    return ($user, $name);
}

# Token feature (see oar.conf)
# Keep the track of the usage for each token
my %Token_values;

oar_info($scheduler_name, "Begin phase 1 (already scheduled jobs)\n", $Session_id);

# Take care of currently scheduled jobs (gantt in the database)
my ($order, %already_scheduled_jobs) = OAR::IO::get_gantt_scheduled_jobs($base);
my @already_scheduled_jobs_list = @{$order};
while (@already_scheduled_jobs_list) {
    my $i         = shift(@already_scheduled_jobs_list);
    my $types     = OAR::IO::get_job_types_hash($base, $i);
    my @type_list = sort(keys(%{$types}));

    # Do not take care of besteffort jobs
    if ((!defined($types->{besteffort})) or ($queue eq "besteffort")) {
        my @resource_list     = @{ $already_scheduled_jobs{$i}->[3] };
        my $resource_list_vec = $already_scheduled_jobs{$i}->[10];
        my $job_duration      = $already_scheduled_jobs{$i}->[1];
        if ($already_scheduled_jobs{$i}->[4] eq "Suspended") {

            # Remove resources of the type specified in SCHEDULER_AVAILABLE_SUSPENDED_RESOURCE_TYPE
            ($resource_list_vec, @resource_list) = OAR::IO::get_job_current_resources(
                $base,
                $already_scheduled_jobs{$i}->[7],
                \@Sched_available_suspended_resource_type);
            next if ($#resource_list < 0);
        }
        if ($already_scheduled_jobs{$i}->[8] eq "YES") {

            # This job was suspended so we must recalculate the walltime
            $job_duration += OAR::IO::get_job_suspended_sum_duration($base, $i, $current_time);
        }

        my $gantt_to_use = $Container_gantt_hash{0};
        if (defined($types->{container})) {
            oar_info($scheduler_name, "container job: create a new gantt for container $i\n",
                $Session_id, "$i");
            $Container_gantt_hash{$i} =
              OAR::Schedulers::GanttHoleStorage_with_quotas::new_with_1_hole(
                $Max_resources, $minimum_hole_time,
                $already_scheduled_jobs{$i}->[0],
                $job_duration + $security_time_overhead,
                $resource_list_vec, $All_resource_list_vec);
            $gantt_to_use = $Container_gantt_hash{$i};
        } else {
            OAR::Schedulers::QuotaStorage::update_accounting_counters(
                $quota_accounting_data,
                unpack("%32b*", $resource_list_vec & $Only_default_type_resource_vec),
                $already_scheduled_jobs{$i}->[0],
                $job_duration + $security_time_overhead,
                $already_scheduled_jobs{$i}->[2],
                $already_scheduled_jobs{$i}->[9],
                \@type_list,
                $already_scheduled_jobs{$i}->[5]);
        }

        #Handle a job within a container
        my $container_num = 0;
        if (defined($types->{inner}) and ($types->{inner} =~ /^(\d+)$/)) {
            if (defined($Container_gantt_hash{$1})) {
                $gantt_to_use  = $Container_gantt_hash{$1};
                $container_num = $1;
                oar_info($scheduler_name, "inner job: using container $container_num\n",
                    $Session_id, "$i");
            } else {
                if (grep(/^$1$/, @already_scheduled_jobs_list)) {

                    # The container job is later in the list, postpone inner job
                    oar_info($scheduler_name,
                        "inner job: container $1 is not known yet, postponing job.\n",
                        $Session_id, "$i");
                    push @already_scheduled_jobs_list, $i;
                    next;
                } else {
                    oar_info(
                        $scheduler_name,
                        "inner job: container $1 does not exist but job is already scheduled or running, use container 0.\n",
                        $Session_id,
                        "$i");
                }
            }
        }
        my $user;
        my $name;
        if (defined($types->{timesharing})) {
            ($user, $name) = parse_timesharing(
                $types->{timesharing},
                $already_scheduled_jobs{$i}->[5],
                $already_scheduled_jobs{$i}->[6]);
            if (!defined($timesharing_gantts->{$container_num}->{$user}->{$name})) {
                $timesharing_gantts->{$container_num}->{$user}->{$name} = dclone($gantt_to_use);
                oar_info($scheduler_name,
                    "timesharing job: cloned a new gantt for ($container_num, $user, $name)\n",
                    $Session_id, "$i");
            }
        }

        #Fill all other gantts
        foreach my $g (keys(%Container_gantt_hash)) {
            if ($i != $g) {
                oar_info($scheduler_name, "add job occupation in gantt of container $g\n",
                    $Session_id, "$i");
                OAR::Schedulers::GanttHoleStorage_with_quotas::set_occupation(
                    $Container_gantt_hash{$g},
                    $already_scheduled_jobs{$i}->[0],
                    $job_duration + $security_time_overhead,
                    $resource_list_vec);
            }
        }
        foreach my $c (keys(%{$timesharing_gantts})) {
            foreach my $u (keys(%{ $timesharing_gantts->{$c} })) {
                foreach my $n (keys(%{ $timesharing_gantts->{$c}->{$u} })) {
                    if ((!defined($user)) or (!defined($name)) or (($u ne $user) or ($n ne $name)))
                    {
                        oar_info($scheduler_name,
                            "add job occupation in gantt of timesharing ($c, $u, $n)\n",
                            $Session_id, "$i");
                        OAR::Schedulers::GanttHoleStorage_with_quotas::set_occupation(
                            $timesharing_gantts->{$c}->{$u}->{$n},
                            $already_scheduled_jobs{$i}->[0],
                            $job_duration + $security_time_overhead,
                            $resource_list_vec);
                    } else {
                        oar_info(
                            $scheduler_name,
                            "skip timesharing job occupation in gantt of timesharing ($c, $u, $n)\n",
                            $Session_id,
                            "$i");
                    }
                }
            }
        }
    }
}

oar_info($scheduler_name, "End phase 1 (already scheduled jobs)\n", $Session_id);

# End of the initialisation
# Begining of the real scheduling

# Get list of Alive resources
my ($Alive_resource_id_vec,     undef) = OAR::IO::get_resource_ids_in_state($base, "Alive");
my ($Absent_resource_id_vec,    undef) = OAR::IO::get_resource_ids_in_state($base, "Absent");
my ($Suspected_resource_id_vec, undef) = OAR::IO::get_resource_ids_in_state($base, "Suspected");
my ($Dead_resource_id_vec, @Dead_resource_ids) = OAR::IO::get_resource_ids_in_state($base, "Dead");

# ENERGY SAVING: add fake occupations/holes from energy saving configuration
# CM part and Hulot part (wake up nodes in energy saving mode)
if (is_conf("SCHEDULER_NODE_MANAGER_WAKE_UP_CMD") or
    (get_conf("ENERGY_SAVING_INTERNAL") eq "yes" and
        is_conf("ENERGY_SAVING_NODE_MANAGER_WAKE_UP_CMD"))
) {
    oar_info($scheduler_name, "Begin EnergySaving phase\n", $Session_id);

    # Check the resources that can be waked_up or shut down
    my $upto_availability = OAR::IO::get_energy_saving_resources_availability($base, $current_time);
    foreach my $t (keys(%{$upto_availability})) {
        my $vec = '';
        foreach my $r (@{ $upto_availability->{$t} }) {
            vec($Alive_resource_id_vec, $r, 1) = 1;
            vec($vec,                   $r, 1) = 1;
        }

        #Fill all the gantts
        foreach my $g (keys(%Container_gantt_hash)) {
            oar_info(
                $scheduler_name,
                "Add EnergySaving occupation in gantt for container $g with available_upto = $t\n",
                $Session_id);
            OAR::Schedulers::GanttHoleStorage_with_quotas::set_occupation($Container_gantt_hash{$g},
                $t, OAR::Schedulers::GanttHoleStorage_with_quotas::get_infinity_value(), $vec);
        }

        foreach my $c (keys(%{$timesharing_gantts})) {
            foreach my $u (keys(%{ $timesharing_gantts->{$c} })) {
                foreach my $n (keys(%{ $timesharing_gantts->{$c}->{$u} })) {
                    oar_info(
                        $scheduler_name,
                        "Add EnergySaving occupation in gantt for timesharing ($c, $u, $n) with available_upto = $t\n",
                        $Session_id);
                    OAR::Schedulers::GanttHoleStorage_with_quotas::set_occupation(
                        $timesharing_gantts->{$c}->{$u}->{$n},
                        $t, OAR::Schedulers::GanttHoleStorage_with_quotas::get_infinity_value(),
                        $vec);
                }
            }
        }
    }
    oar_info($scheduler_name, "End EnergySaving phase\n", $Session_id);
}

# CM part

oar_info($scheduler_name, "Begin phase 2 (waiting jobs)\n", $Session_id);
my @jobs = OAR::IO::get_fairsharing_jobs_to_schedule($base, $queue,
    $Karma_max_number_of_jobs_treated_per_user);
###############################################################################
# Sort jobs depending on their previous usage
# Karma sort algorithm
my $Karma_sum_time =
  OAR::IO::get_sum_accounting_window($base, $queue, $current_time - $Karma_window_size,
    $current_time);
$Karma_sum_time->{ASKED} = 1 if (!defined($Karma_sum_time->{ASKED}));
$Karma_sum_time->{USED}  = 1 if (!defined($Karma_sum_time->{USED}));

my $Karma_projects =
  OAR::IO::get_sum_accounting_for_param($base, $queue, "accounting_project",
    $current_time - $Karma_window_size,
    $current_time);
my $Karma_users =
  OAR::IO::get_sum_accounting_for_param($base, $queue, "accounting_user",
    $current_time - $Karma_window_size,
    $current_time);

sub karma($) {
    my $j = shift;

    my $note = 0;
    $note =
      $Karma_coeff_project_consumption *
      (($Karma_projects->{ $j->{project} }->{USED} / $Karma_sum_time->{USED}) -
          ($Karma_project_targets->{ $j->{project} } / 100));
    $note +=
      $Karma_coeff_user_consumption *
      (($Karma_users->{ $j->{job_user} }->{USED} / $Karma_sum_time->{USED}) -
          ($Karma_user_targets->{ $j->{job_user} } / 100));
    $note +=
      $Karma_coeff_user_asked_consumption *
      (($Karma_users->{ $j->{job_user} }->{ASKED} / $Karma_sum_time->{ASKED}) -
          ($Karma_user_targets->{ $j->{job_user} } / 100));

    return ($note);
}

###############################################################################

@jobs = sort({ karma($a) <=> karma($b) } @jobs);
while (@jobs and ((time() - $initial_time) < $timeout)) {
    my $j = shift(@jobs);
    oar_info($scheduler_name, "start scheduling (Karma note = " . karma($j) . ")\n",
        $Session_id, "$j->{job_id}");

    my $gantt_to_use  = $Container_gantt_hash{0};
    my $container_num = 0;
    my $types         = OAR::IO::get_job_types_hash($base, $j->{job_id});
    my @type_list     = sort(keys(%{$types}));

    my $scheduler_init_date = $current_time;

    if (defined($types->{postpone})) {
        my $postpone_time_sec = OAR::IO::sql_to_local($types->{postpone});
        if ($scheduler_init_date < $postpone_time_sec) {
            $scheduler_init_date = $postpone_time_sec;
            oar_info($scheduler_name, "job is postponed to $types->{postpone}\n",
                $Session_id, "$j->{job_id}");
        }
    }

    if (defined($types->{expire})) {
        my $expire_time_sec = OAR::IO::sql_to_local($types->{expire});
        if ($scheduler_init_date > $expire_time_sec) {
            my $message = "job will never run (expire=$types->{expire}), setting it to error";
            oar_info($scheduler_name, "$message\n", $Session_id, "$j->{job_id}");
            OAR::IO::set_job_message($base, $j->{job_id}, ucfirst($message));
            OAR::IO::set_job_state($base, $j->{job_id}, "toError");
            next;
        }
    }

    my $deadline_time_sec;
    if (defined($types->{deadline})) {
        $deadline_time_sec = OAR::IO::sql_to_local($types->{deadline});
        if ($scheduler_init_date > $deadline_time_sec) {
            my $message = "job will never run (deadline=$types->{deadline}), setting it to error";
            oar_info($scheduler_name, "$message\n", $Session_id, "$j->{job_id}");
            OAR::IO::set_job_message($base, $j->{job_id}, ucfirst($message));
            OAR::IO::set_job_state($base, $j->{job_id}, "toError");
            next;
        }
    }

    # Search for dependencies
    my $skip_job = 0;
    foreach my $d (OAR::IO::get_current_job_dependencies($base, $j->{job_id})) {
        my $dep_job = OAR::IO::get_job($base, $d);
        if (($dep_job->{state} ne "Terminated") and ($dep_job->{state} ne "Error")) {
            my @date_tmp = OAR::IO::get_gantt_job_start_time($base, $d);
            if (defined($date_tmp[0])) {
                my $mold_dep  = OAR::IO::get_current_moldable_job($base, $date_tmp[1]);
                my $sched_tmp = $date_tmp[0] + $mold_dep->{moldable_walltime};
                if ($scheduler_init_date < $sched_tmp) {
                    $scheduler_init_date = $sched_tmp + (2 * $security_time_overhead);
                }
            } else {
                my $message = "cannot determine scheduling time due to dependency with the job $d";
                OAR::IO::set_job_message($base, $j->{job_id}, ucfirst($message));
                oar_info($scheduler_name, "$message\n", $Session_id, "$j->{job_id}");
                $skip_job = 1;
                last;
            }
        }
    }
    next if ($skip_job == 1);

    if (defined($types->{inner}) and ($types->{inner} =~ /^(\d+)$/)) {
        if ($1 == $j->{job_id}) {
            my $message = "inner job: job cannot be its own container, setting it to error";
            oar_warn($scheduler_name, "$message\n", $Session_id, "$j->{job_id}");
            OAR::IO::set_job_message($base, $j->{job_id}, ucfirst($message));
            OAR::IO::set_job_state($base, $j->{job_id}, "toError");
            next;
        }
        if (defined($Container_gantt_hash{$1})) {
            $gantt_to_use  = $Container_gantt_hash{$1};
            $container_num = $1;
            oar_info($scheduler_name, "inner job: using container $container_num\n",
                $Session_id, "$j->{job_id}");
        } else {

            #is the container yet to schedule, by this scheduler or another ?
            if (grep(/^$1$/, map { $_->{job_id} } @jobs)) {

                # The container job is later in the list, postpone inner job
                oar_info($scheduler_name,
                    "inner job: container $1 is not known yet, postponing job.\n",
                    $Session_id, "$j->{job_id}");
                push @jobs, $j;
            } elsif (grep (/^$1$/, OAR::IO::get_all_waiting_jobids($base))) {
                my $message =
                  "inner job: container $1 is yet to scheduled, inner job not scheduled";
                oar_info($scheduler_name, "$message\n", $Session_id, "$j->{job_id}");
                OAR::IO::set_job_message($base, $j->{job_id}, ucfirst($message));

                #OAR::IO::set_job_scheduler_info($base,$j->{job_id},"Container $1 does not exist");
            } else {
                my $message =
                  "inner job: container $1 does not exist, inner job will never run, setting it to error";
                oar_warn($scheduler_name, "$message\n", $Session_id, "$j->{job_id}");
                OAR::IO::set_job_message($base, $j->{job_id}, ucfirst($message));
                OAR::IO::set_job_state($base, $j->{job_id}, "toError");
            }
            next;
        }
    }
    if (defined($types->{timesharing})) {
        my ($user, $name) =
          parse_timesharing($types->{timesharing}, $j->{job_user}, $j->{job_name});
        if (!defined($timesharing_gantts->{$container_num}->{$user}->{$name})) {
            $timesharing_gantts->{$container_num}->{$user}->{$name} = dclone($gantt_to_use);
            oar_info($scheduler_name,
                "timesharing job: cloned a new gantt for ($container_num, $user, $name)\n",
                $Session_id, "$j->{job_id}");
        }
        $gantt_to_use = $timesharing_gantts->{$container_num}->{$user}->{$name};
        oar_info($scheduler_name, "timesharing job: use gantt for ($container_num, $user, $name)\n",
            $Session_id, "$j->{job_id}");
    }
    if (defined($types->{token})) {
        foreach my $t (keys(%{ $types->{token} })) {
            next if ($skip_job == 1);
            oar_info($scheduler_name, "check token: $t ($types->{token}->{$t}).\n",
                $Session_id, "$j->{job_id}");

            # Check if we must execute the corresponding script
            if ((!defined($Token_values{$t})) and (defined($Token_scripts->{$t}))) {
                oar_info($scheduler_name, "execute $Token_scripts->{$t}\n",
                    $Session_id, "$j->{job_id}");
                if (open(TOKSCRIPT, "$Token_scripts->{$t} |")) {
                    my $num = <TOKSCRIPT>;
                    chop($num);
                    if ($num =~ /^\d+$/) {
                        $Token_values{$t} = $num;
                        oar_info($scheduler_name, "$Token_scripts->{$t} returns $num\n",
                            $Session_id, "$j->{job_id}");
                    } else {
                        oar_warn(
                            $scheduler_name,
                            "the token script $Token_scripts->{$t} does not return a number ($num).\n",
                            $Session_id,
                            "$j->{job_id}");
                    }
                    close(TOKSCRIPT);
                }
            }
            if (defined($Token_values{$t})) {
                if ($Token_values{$t} < $types->{token}->{$t}) {
                    my $message =
                      "no enough Tokens: $t ($Token_values{$t} < $types->{token}->{$t})";
                    oar_info($scheduler_name, "$message\n", $Session_id, "$j->{job_id}");
                    OAR::IO::set_job_message($base, $j->{job_id}, ucfirst($message));

#OAR::IO::set_job_scheduler_info($base,$j->{job_id},"No enough Token: $t ($Token_values{$t} < $types->{token}->{$t})");
                    $skip_job = 1;
                }
            } else {
                my $message = "Token value cannot be retrieved ($t)";
                oar_info($scheduler_name, "$message\n", $Session_id, "$j->{job_id}");
                OAR::IO::set_job_message($base, $j->{job_id}, ucfirst($message));

        #OAR::IO::set_job_scheduler_info($base,$j->{job_id},"Token value cannot be retrieved ($t)");
                $skip_job = 1;
            }
        }
        next if ($skip_job == 1);
    }

#OAR::Schedulers::GanttHoleStorage_with_quotas::pretty_print($gantt_to_use);
#oar_info("[$scheduler_name] [$j->{job_id}] Gantt data structure:\n".OAR::Schedulers::GanttHoleStorage_with_quotas::pretty_print($gantt_to_use));

    my $available_resources_vector = $Alive_resource_id_vec;

    # Allow for scheduling jobs on absent or suspected resources is job has type state=permissive
    if (defined($types->{state}) and $types->{state} eq "permissive") {
        $available_resources_vector |= $Absent_resource_id_vec | $Suspected_resource_id_vec;
    }
    my $job_properties = "\'1\'";
    if ((defined($j->{properties})) and ($j->{properties} ne "")) {
        $job_properties = $j->{properties};
    }

    # Choose the moldable job to schedule
    my @moldable_results;
    my $job_descriptions = OAR::IO::get_resources_data_structure_current_job($base, $j->{job_id});
    foreach my $moldable (@{$job_descriptions}) {
        my $duration;
        if (defined($types->{besteffort})) {
            $duration = $besteffort_duration;
        } else {
            $duration = $moldable->[1] + $security_time_overhead;
        }

        my @tree_list;
        foreach my $m (@{ $moldable->[0] }) {
            my $tmp_properties = "\'1\'";
            if ((defined($m->{property})) and ($m->{property} ne "")) {
                $tmp_properties = $m->{property};
            }
            my $tmp_tree =
              OAR::IO::get_possible_wanted_resources($base_ro, $available_resources_vector, undef,
                \@Dead_resource_ids, "$job_properties AND $tmp_properties",
                $m->{resources},     $Order_part);
            push(@tree_list, $tmp_tree);
        }
        my $gantt_timeout = ($timeout - (time() - $initial_time)) / 4;
        $gantt_timeout = $Minimum_timeout_per_job if ($gantt_timeout <= ($timeout / 8));
        oar_info($scheduler_name, "find_first_hole with a timeout of $gantt_timeout\n",
            $Session_id, "$j->{job_id}");
        my @hole = OAR::Schedulers::GanttHoleStorage_with_quotas::find_first_hole(
            $gantt_to_use,          $scheduler_init_date,
            $duration,              \@tree_list,
            $gantt_timeout,         $j->{queue_name},
            $j->{project},          \@type_list,
            $j->{job_user},         $Gantt_quotas,
            $quota_accounting_data, $Only_default_type_resource_vec);

        # Free memory
        foreach my $t (@tree_list) {
            OAR::Schedulers::ResourceTree::destroy($t);
        }
        my @resources;
        my $res_vec = '';
        foreach my $t (@{ $hole[2] }) {
            $res_vec |= $t->[1];
            push(@resources, @{ $t->[2] });
            OAR::Schedulers::ResourceTree::destroy($t->[0]);
        }
        push(
            @moldable_results,
            {   resources     => \@resources,
                resources_vec => $res_vec,
                start_date    => $hole[0],
                duration      => $duration,
                moldable_id   => $moldable->[2],
                walltime      => $moldable->[1],
                comment       => $hole[1] });
    }

    # Choose moldable job which will finish the first
    my $index_to_choose = -1;
    my $best_stop_time;
    for (my $i = 0; $i <= $#moldable_results; $i++) {
        if ($#{ $moldable_results[$i]->{resources} } >= 0) {
            my $tmp_stop_date =
              $moldable_results[$i]->{start_date} + $moldable_results[$i]->{duration};
            if ((!defined($best_stop_time)) or ($best_stop_time > $tmp_stop_date)) {
                $best_stop_time  = $tmp_stop_date;
                $index_to_choose = $i;
            }
        }
    }
    if ($index_to_choose >= 0) {
        if (defined($deadline_time_sec) and $best_stop_time > $deadline_time_sec) {
            oar_info(
                $scheduler_name,
                "not scheduling job because its deadline is not met: $best_stop_time > $deadline_time_sec\n",
                $Session_id,
                "$j->{job_id}");
            next;
        }

        # We can schedule the job
        #Fill all other gantts
        foreach my $g (keys(%Container_gantt_hash)) {
            if ($j->{job_id} != $g) {
                oar_info($scheduler_name, "add job occupation in gantt of container $g\n",
                    $Session_id, "$j->{job_id}");
                OAR::Schedulers::GanttHoleStorage_with_quotas::set_occupation(
                    $Container_gantt_hash{$g},
                    $moldable_results[$index_to_choose]->{start_date},
                    $moldable_results[$index_to_choose]->{duration},
                    $moldable_results[$index_to_choose]->{resources_vec});
            }
        }
        if (defined($types->{container})) {
            $Container_gantt_hash{ $j->{job_id} } =
              OAR::Schedulers::GanttHoleStorage_with_quotas::new_with_1_hole(
                $Max_resources,
                $minimum_hole_time,
                $moldable_results[$index_to_choose]->{start_date},
                $moldable_results[$index_to_choose]->{duration},
                $moldable_results[$index_to_choose]->{resources_vec},
                $All_resource_list_vec);
        } else {
            OAR::Schedulers::QuotaStorage::update_accounting_counters(
                $quota_accounting_data,
                unpack(
                    "%32b*",
                    $moldable_results[$index_to_choose]->{resources_vec} &
                      $Only_default_type_resource_vec
                ),
                $moldable_results[$index_to_choose]->{start_date},
                $moldable_results[$index_to_choose]->{duration},
                $j->{queue_name},
                $j->{project},
                \@type_list,
                $j->{job_user});

#oar_info("[$scheduler_name] [$j->{job_id}] ".OAR::Schedulers::QuotaStorage::pretty_print($quota_accounting_data));
        }

        foreach my $c (keys(%{$timesharing_gantts})) {
            foreach my $u (keys(%{ $timesharing_gantts->{$c} })) {
                foreach my $n (keys(%{ $timesharing_gantts->{$c}->{$u} })) {
                    if (($gantt_to_use != $timesharing_gantts->{$c}->{$u}->{$n})) {
                        oar_info($scheduler_name,
                            "add job occupation in gantt of timesharing ($c, $u, $n)\n",
                            $Session_id, "$j->{job_id}");
                        OAR::Schedulers::GanttHoleStorage_with_quotas::set_occupation(
                            $timesharing_gantts->{$c}->{$u}->{$n},
                            $moldable_results[$index_to_choose]->{start_date},
                            $moldable_results[$index_to_choose]->{duration},
                            $moldable_results[$index_to_choose]->{resources_vec});
                    } else {
                        oar_info(
                            $scheduler_name,
                            "skip timesharing job occupation in gantt of timesharing ($c, $u, $n)\n",
                            $Session_id,
                            "$j->{job_id}");
                    }
                }
            }
        }

        #update database
        push(@{ $moldable_results[$index_to_choose]->{resources} }, @Resources_to_always_add);
        OAR::IO::add_gantt_scheduled_jobs(
            $base,
            $moldable_results[$index_to_choose]->{moldable_id},
            $moldable_results[$index_to_choose]->{start_date},
            $moldable_results[$index_to_choose]->{resources});

        # Feed message job field
        my $message = OAR::IO::format_job_message_text(
            $j->{job_name},
            $#{ $moldable_results[$index_to_choose]->{resources} } + 1,
            $moldable_results[$index_to_choose]->{walltime},
            $j->{job_type},
            $j->{reservation},
            $j->{queue_name},
            $j->{project},
            \@type_list,
            "Karma=" .
              sprintf("%.3f", karma($j)) . ",$moldable_results[$index_to_choose]->{comment}");
        OAR::IO::set_job_message($base, $j->{job_id}, ucfirst($message));

        #OAR::IO::set_job_scheduler_info($base,$j->{job_id},"Karma = ".sprintf("%.3f",karma($j)));

        # Update Token values
        if (defined($types->{token}) and
            ($moldable_results[$index_to_choose]->{start_date} <= $current_time)) {
            foreach my $t (keys(%{ $types->{token} })) {
                $Token_values{$t} = $Token_values{$t} - $types->{token}->{$t};
                oar_info($scheduler_name, "Update TOKEN $t to $Token_values{$t}\n", $Session_id);
            }
        }

        if ($moldable_results[$index_to_choose]->{start_date} <= $current_time) {

            # Try tu run the job now
            print(
                "SCHEDRUN JOB_ID=$j->{job_id} MOLDABLE_JOB_ID=$moldable_results[$index_to_choose]->{moldable_id} RESOURCES="
                  . join(',', @{ $moldable_results[$index_to_choose]->{resources} })
                  . "\n");
        }
    } elsif ($moldable_results[0]->{comment} ne "TIMEOUT") {
        my $message = "no enough matching resources ($moldable_results[0]->{comment})";
        OAR::IO::set_job_message($base, $j->{job_id}, ucfirst($message));

        #OAR::IO::set_job_scheduler_info($base,$j->{job_id},$message);
        oar_info($scheduler_name, "$message\n", $Session_id, "$j->{job_id}");
    }
    oar_info($scheduler_name, "end scheduling\n", $Session_id, "$j->{job_id}");
}
oar_info($scheduler_name, "End phase 2 (waiting jobs)\n", $Session_id);

OAR::IO::disconnect($base);
OAR::IO::disconnect($base_ro);

if (@jobs) {
    oar_info(
        $scheduler_name,
        "Warning: some jobs were not scheduled because the scheduler's timeout was reached ($timeout s)\n",
        $Session_id);
}

oar_info($scheduler_name, "End of scheduler for queue $queue\n", $Session_id);

