sreview-master 0.3.0-1 / usr / bin / sreview-dispatch

#!/usr/bin/perl -w

# Sreview, a web-based video review and transcoding system
# Copyright (c) 2016-2017, Wouter Verhelst <w@uter.be>
#
# Sreview is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License along with this program.  If not, see
# <http://www.gnu.org/licenses/>.

use strict;
use warnings;

use DBI;
use Mojo::Template;
use SReview::Config::Common;

=head1 NAME

sreview-dispatch - Watch the database for talks that need work done, and schedule jobs for them

=head1 SYNOPSIS

sreview-dispatch

=head1 DESCRITPION

B<sreview-dispatch> is the central script for SReview. It can be used in two ways:

=over

=item 1.

Either you run it with an external scheduler (e.g., gridengine, slurm,
PBS, torque). This is the recommended way of using SReview for large
installations. In this mode of operation, C<sreview-dispatch> should be
run once in the entire network, and the C<query_limit> configuration
value should be set to 0.

=item 2.

Or you run it with no external scheduler. This is the default. In this
mode of operation, it is recommended that the C<query_limit>
configuration parameter is set to a nonzero value, so that individual
C<sreview-dispatch> instances do not take all the work, keeping all the
other instances idle. In this mode of operation, one C<sreview-dispatch>
instance should be run per CPU core on every machine that is used for
performing work.

This mode works, but using an external scheduler allows for operating
more flexibly on pending jobs, and on the addition or removal of extra
nodes.

=back

=head1 OPTIONS

None. C<sreview-dispatch> uses the system-wide SReview configuration.
For more information, see L<sreview-config>

=head1 CONFIGURATION

C<sreview-dispatch> considers the following configuration values:

=over

=cut

my $config = SReview::Config::Common::setup;

=item dbistring

The database connection string

=cut

my $dbh = DBI->connect($config->get('dbistring'), '', '') or die "Cannot connect to database!";

=item state_actions

A hash table of shell commands to execute when a talk is found in a
particular state with the C<waiting> progress value. The hash key should
be the state name.

If a state is not specified in the C<state_actions> hash, then
C<sreview-dispatch> will I<ignore> that state. This allows for handling
particular states on particular hosts, if necessary.

The shell commands can use the following template variables:

=over

=item <%== $talkid %>

The database ID of the talk to work on

=item <%== $output_dir %>

The value of the C<script_output> configuration item, i.e., where to
redirect output to.

=back

=cut

my $state_actions = $config->get('state_actions');

my $mt = Mojo::Template->new;

$mt->vars(1);

while(1) {
	$dbh->begin_work;
	print "==> Checking for new work...\n";
	my $next = $dbh->prepare("UPDATE talks SET state = state_next(state), progress='waiting' WHERE id = ?");
	my $start = $dbh->prepare("UPDATE talks SET progress = 'scheduled' WHERE id = ?");

	my $st;
	my $statelist = "'" . join("','", keys(%{$state_actions})) . "'";

=item query_limit

The maximum number of requests that should be handled per loop. Should
be set to 0 (meaning infinity) when using an external scheduler;
should probably be set to 1 when not.

=cut

	if($config->get('query_limit') > 0) {
		$st = $dbh->prepare("SELECT talks.id, state, progress, title, rooms.name AS room, extract(epoch from (endtime - starttime)) AS length FROM talks JOIN rooms ON rooms.id = talks.room WHERE (state IN ($statelist) AND progress = 'waiting') OR progress = 'done' LIMIT ? FOR UPDATE");
		$st->execute($config->get('query_limit'));
	} else {
		$st = $dbh->prepare("SELECT talks.id, state, progress, title, rooms.name AS room, extract(epoch from (endtime - starttime)) AS length FROM talks JOIN rooms ON rooms.id = talks.room WHERE (state IN ($statelist) AND progress = 'waiting') OR progress = 'done' FOR UPDATE");
		$st->execute;
	}
	while(my $row = $st->fetchrow_hashref) {
		if($row->{progress} eq "done") {
			print "Job for event " . $row->{title} . " in state " . $row->{state} . " finished, migrating it to the next state\n";
			$next->execute($row->{id});
			next;
		}
		if(exists(${$state_actions}{$row->{state}})) {
			print "Starting job for event " . $row->{title} . " in state " . $row->{state} . "...\n";
			$start->execute($row->{id});
			my $statetrans = ${$state_actions}{$row->{state}};

=item script_output

The location where stderr and stdout output should be written to. This
is supposed to be a directory; it is reasonable to create a file for
stderr and a file for stdout for each script run.

=back

=cut

			system $mt->render($statetrans, {talkid => $row->{id}, room => $row->{room}, length => $row->{length}, output_dir => $config->get('script_output')});
		}
	}
	print "finished, waiting 10 seconds...\n";
	$dbh->commit;

	sleep 10;
}

=head1 BUGS

C<sreview-dispatch> currently polls the database every 10 seconds. This
should be changed to an event-based system based on PostgreSQL
asynchronous notifications.

=cut