xmltv-util 0.5.69-1 / usr / bin / tv_grab_eu_dotmedia

#!/usr/bin/perl -w

=pod

=head1 NAME

=encoding utf8

tv_grab_eu_dotmedia - Grab TV listings for Europe.

=head1 SYNOPSIS

tv_grab_eu_dotmedia --help

tv_grab_eu_dotmedia --configure [--config-file FILE] [--gui OPTION]

tv_grab_eu_dotmedia [--config-file FILE]
           [--days N] [--offset N]
           [--output FILE] [--quiet] [--debug]

tv_grab_eu_dotmedia --list-channels [--config-file FILE]
           [--output FILE] [--quiet] [--debug]


=head1 DESCRIPTION

Output TV and listings in XMLTV format for many stations
available in Europe.

First you must run B<tv_grab_eu_dotmedia --configure> to choose which stations
you want to receive.

Then running B<tv_grab_eu_dotmedia> with no arguments will get a listings for
the stations you chose for five days including today.

=head1 OPTIONS

B<--configure> Prompt for which stations to download and write the
configuration file.

B<--config-file FILE> Set the name of the configuration file, the
default is B<~/.xmltv/tv_grab_eu_dotmedia.conf>.  This is the file written by
B<--configure> and read when grabbing.

B<--gui OPTION> Use this option to enable a graphical interface to be used.
OPTION may be 'Tk', or left blank for the best available choice.
Additional allowed values of OPTION are 'Term' for normal terminal output
(default) and 'TermNoProgressBar' to disable the use of Term::ProgressBar.

B<--output FILE> When grabbing, write output to FILE rather than
standard output.

B<--days N> When grabbing, grab N days rather than 5.

B<--offset N> Start grabbing at today + N days.  N may be negative.

B<--quiet> Suppress the progress-bar normally shown on standard error.

B<--debug> Provide more information on progress to stderr to help in
debugging.

B<--list-channels>    Output a list of all channels that data is available
                      for. The list is in xmltv-format.

B<--version> Show the version of the grabber.

B<--help> Print a help message and exit.

=head1 ERROR HANDLING

If the grabber fails to download data for some channel on a specific day,
it will print an errormessage to STDERR and then continue with the other
channels and days. The grabber will exit with a status code of 1 to indicate
that the data is incomplete.

=head1 ENVIRONMENT VARIABLES

The environment variable HOME can be set to change where configuration
files are stored. All configuration is stored in $HOME/.xmltv/. On Windows,
it might be necessary to set HOME to a path without spaces in it.

=head1 SUPPORTED CHANNELS

For information on supported channels, see http://xmltv.xmltv.se

=head1 AUTHOR

Joakim Nylén, me -at- jnylen -dot- nu. This script is a modified version
of tv_grab_se_swedb by Mattias Holmlund, mattias -at- holmlund -dot- se.
While the documentation and parts of the code copied from tv_grab_uk by
Ed Avis, ed -at- membled -dot- com.

=head1 BUGS

=cut

use strict;

use XMLTV;
use XMLTV::ProgressBar;
use XMLTV::Options qw/ParseOptions/;
use XMLTV::Configure::Writer;

use XML::LibXML;
use Date::Manip;
use Compress::Zlib;
use File::Path;
use File::Basename;
use IO::Scalar;
use LWP;
use LWP::Simple;
use Data::Dumper;

my $ua;
$ua = LWP::UserAgent->new();
$ua->agent("xmltv/$XMLTV::VERSION");
$ua->env_proxy();

use HTTP::Cache::Transparent;

# Although we use HTTP::Cache::Transparent, this undocumented --cache
# option for debugging is still useful since it will _always_ use a
# cached copy of a page, without contacting the server at all.
#
use XMLTV::Memoize; XMLTV::Memoize::check_argv('getuncompressed');

sub t;

my $default_root_url = 'http://xmltv.xmltv.se/';
my $default_main_url = 'http://xmltv.se/';
my $default_cachedir = get_default_cachedir();

my( $opt, $conf ) = ParseOptions( {
    grabber_name => "tv_grab_eu_dotmedia",
    capabilities => [qw/baseline manualconfig tkconfig apiconfig cache/],
    stage_sub => \&config_stage,
    listchannels_sub => \&list_channels,
    load_old_config_sub => \&load_old_config,
    version => '$Id: tv_grab_eu_dotmedia,v 1.3 2016/03/21 12:33:12 knowledgejunkie Exp $',
    description => "Europe tv schedules for free",

} );

if (not defined( $conf->{cachedir} )) {
    print STDERR "No cachedir defined in configfile " .
                 $opt->{'config-file'} . "\n" .
                 "Please run the grabber with --configure.\n";
    exit 1;
}

if (not defined( $conf->{'country'} )) {
    print STDERR "No country defined in configfile " .
                 $opt->{'config-file'} . "\n" .
                 "Please run the grabber with --configure.\n";
    exit 1;
}

if (not defined( $conf->{'channel'} )) {
    print STDERR "No channels selected in configfile " .
                 $opt->{'config-file'} . "\n" .
                 "Please run the grabber with --configure.\n";
    exit 1;
}

init_cachedir( $conf->{cachedir}->[0] );
HTTP::Cache::Transparent::init( {
    BasePath => $conf->{cachedir}->[0],
    NoUpdate => 15*60,
    Verbose => $opt->{debug},
    } );

binmode (STDOUT);

my($xmldecl, $channels) = load_channels( $conf->{'root-url'}->[0] );

my( $odoc, $root );
my $warnings = 0;

write_header( $xmldecl );

write_channel_list( $conf->{channel} );

my $now = ParseDate( 'now' );
my $date =$now;
$date = DateCalc( $now, "+$opt->{offset} days" )
    if( $opt->{offset} );

my $bar = undef;
$bar = new XMLTV::ProgressBar( {
    name => 'downloading listings',
    count => $opt->{days} * @{$conf->{channel}},
    }) if (not $opt->{quiet}) && (not $opt->{debug});

for( my $i=0; $i < $opt->{days}; $i++ )
{
    t "Date: $date";
    foreach my $channel_id (@{$conf->{channel}})
    {
        # We have already warned the user if the channel doesn't exist.
        if( exists $channels->{$channel_id} )
        {
            t "  $channel_id";
            my( $channel_name, $url ) = @{$channels->{$channel_id}};
            print_data( $url, $channel_id, $date )
                or warning( "Failed to download data for $channel_id on " .
                            UnixDate( $date, "%Y-%m-%d" ) . "." );
        }
        $bar->update() if defined( $bar );
    }
    $date = DateCalc( $date, "+1 days" );
}

$bar->finish() if defined $bar;

write_footer();

# Signal that something went wrong if there were warnings.
exit(1) if $warnings;

# All data fetched ok.
t "Exiting without warnings.";
exit(0);

sub t
{
    my( $message ) = @_;
    print STDERR $message . "\n" if $opt->{debug};
}

sub warning
{
    my( $message ) = @_;
    print STDERR $message . "\n";
    $warnings++;
}

sub list_channels
{
    my( $conf, $opt ) = @_;

    ( $xmldecl, $channels ) = load_channels( $conf->{'country'}->[0] );

    my $result="";
    my $fh = new IO::Scalar \$result;
    my $oldfh = select( $fh );
    write_header( $xmldecl );
    write_channel_list( [sort keys %{$channels}] );
    write_footer();
    select( $oldfh );
    $fh->close();

    return $result;
}

sub config_stage
{
    my( $stage, $conf ) = @_;

    die "Unknown stage $stage" if $stage ne "start";

    my $result;
    my $writer = new XMLTV::Configure::Writer( OUTPUT => \$result,
                                               encoding => 'iso-8859-1' );
    $writer->start( { grabber => 'tv_grab_eu_dotmedia' } );

    my $parser = new XML::LibXML;

    # fetch XML lineup tree
	my $raw = getuncompressed( $default_main_url . 'countries.xml');

	if( !defined $raw ) {
		die 'request failed';
	}

	my $xmlTree = $parser->parse_string( $raw );
	my $root = $xmlTree->getDocumentElement();

	# get each lineup element
	my @kids = $root->childNodes();
	my $size = @kids;

	if( $size == 0 ) {
		die "Couldn't find any countries. Please contact the developer.";
	}

	$writer->start_selectone( {
		id => 'country',
		title => [ [ 'Country', 'en' ] ],
		description => [ [ "Choose one of the countries this grabber provides.", 'en' ] ],
	} );

	# create option for each lineup element
	foreach my $child (@kids)
	{
		my $country = $child -> textContent;

		# Remove empty childs.
		$country =~ s/^\s+|\s+$//g ;
		next if $country eq "";

		my $country_id = $child -> getAttribute('id');

		$writer->write_option( {
			value => $country_id,
			text=> => [ [ $country, 'en' ] ]
		} );
	}

	$writer->end_selectone();

    $writer->write_string( {
        id => 'cachedir',
        title => [ [ 'Directory to store the cache in', 'en' ] ],
        description => [
         [ 'tv_grab_eu_dotmedia uses a cache with files that it has already '.
           'downloaded. Please specify where the cache shall be stored. ',
           'en' ] ],
        default => $default_cachedir,
     } );

    $writer->end( 'select-channels' );

    return $result;
}

#
# Load a configuration file in the old format.
#

sub load_old_config
{
    my( $config_file ) = @_;

    my @lines = XMLTV::Config_file::read_lines( $config_file );

    my $conf = {};
    $conf->{cachedir}->[0] = $default_cachedir;
    $conf->{'country'}->[0] = '';
    $conf->{'root-url'}->[0] = $default_root_url;
    $conf->{channel} = [];

    foreach my $line (@lines)
    {
        next unless defined $line;

        my( $command, $param ) = split( /\s+/, $line, 2 );
        $param =~ tr/\n\r//d;
        $param =~ s/\s+$//;

        if ( $command =~ /^\s*country\s*$/) {
            $conf->{'country'}->[0] = $param;
            $conf->{'root-url'}->[0] = $default_root_url . "channels-" .$param . ".xml.gz";
        } elsif  ( $command =~ /^\s*channel\s*$/) {
            push @{$conf->{channel}}, $param;
        } elsif ( $command eq 'cache-dir' ) {
            $conf->{'cachedir'}->[0] = $param;
        } else {
            die "Unknown command $command in config-file $config_file"
        }
    }

    print Dumper($conf);

    return $conf;
}

sub get_default_cachedir
{
    my $winhome = $ENV{HOMEDRIVE} . $ENV{HOMEPATH}
    if defined( $ENV{HOMEDRIVE} )
        and defined( $ENV{HOMEPATH} );

    my $home = $ENV{HOME} || $winhome || ".";
    return "$home/.xmltv/cache";
}

sub init_cachedir
{
    my( $path ) = @_;
    if( not -d $path )
    {
        mkpath( $path ) or die "Failed to create cache-directory $path: $@";
    }
}

sub load_channels
{
    my( $country ) = @_;

    if (!defined($country)) {
        $country = $conf->{'country'}->[0];
    }

    my $url = $default_root_url . "channels-" .$country . ".xml.gz";


    my %channels;

    my $xmldata = getuncompressed( $url );

    defined( $xmldata ) or die "Failed to fetch $url";

    my $xml = XML::LibXML->new;

    my $doc = $xml->parse_string($xmldata);

    my $xmldecl = "<?xml version='" . $doc->version() . "' " .
        "encoding='" . $doc->encoding() . "'?>\n";

    my $ns = $doc->find( "//channel" );

    foreach my $node ($ns->get_nodelist)
    {
        my $id = $node->findvalue( '@id' );
        my $name = $node->findvalue( 'display-name[1]' );
        my $url = $node->findvalue( 'base-url' );
        my $urlns = $node->find( './base-url' );
        foreach my $urlnode ($urlns->get_nodelist)
        {
            $node->removeChild( $urlnode );
        }
        $channels{$id} = [ $name, $url, $node->toString(0, 1) ];
    }

    return ($xmldecl, \%channels);
}

sub print_data
{
    my( $rooturl, $channel_id, $date ) = @_;

    my $url = $rooturl . $channel_id . "_" . UnixDate( $date, "%Y-%m-%d" ) .
        ".xml.gz";

    my $xmldata = getuncompressed( $url );

    defined $xmldata or return 0;

    my $in = new IO::Scalar \$xmldata;
    while( my $line = $in->getline() )
    {
        last if $line =~ /<tv/;
    }

    while( my $line = $in->getline() )
    {
        last if $line =~ /<\/tv>/;
        print $line;
    }

    return 1;
}

sub write_header
{
    my( $xmldecl ) = @_;

    # Use the same xml declaration as the one in
    # channels.xml
    print $xmldecl;
    print '<!DOCTYPE tv SYSTEM "xmltv.dtd">' . "\n";
    print "<tv>\n";
}

sub write_channel_list
{
    my( $channel_list ) = @_;

    # Write list of channels.
    t 'Writing list of channels.';

    foreach my $channel_id (@{$channel_list})
    {
        if( not exists $channels->{$channel_id} )
        {
            print STDERR "Unknown channel $channel_id." .
                " See http://xmltv.xmltv.se" .
                " for a list of available channels or run" .
                " tv_grab_eu_dotmedia --configure to reconfigure.\n";
            next;
        }

        my( $channel_name, $url, $def ) = @{$channels->{$channel_id}};
        print "  $def\n";
    }
}

sub write_footer
{
    print "</tv>\n";
}

sub getuncompressed {
    my( $url ) = @_;

    my $response = $ua->get($url);

    return undef
        unless $response->is_success;

    my $compressed = $response->content
        or return undef;

    # Since LWP 5.827, the result from get() is already
    # uncompressed.

    my $uncompressed;

    eval {
	$uncompressed = Compress::Zlib::memGunzip( \$compressed );
    };

    $uncompressed = $compressed if not defined $uncompressed;

    return $uncompressed;
}