xmltv-util 0.5.63-2 / usr / bin / tv_grab_fr

#!/usr/bin/perl -w

eval 'exec /usr/bin/perl -w -S $0 ${1+"$@"}'
    if 0; # not running under some shell

=head1 NAME

tv_grab_fr - Grab TV listings for France.

=head1 SYNOPSIS

 To configure: 
   tv_grab_fr --configure [--config-file FILE] [--gui OPTION]
 To configure AUTRES CHAINES list: 
   tv_grab_fr --configure-more-channels
 To grab listings: 
   tv_grab_fr [--config-file FILE] [--output FILE] [--days N] 
    [--offset N] [--quiet] [--debug]
 To list available channels:
   tv_grab_fr --list-channels
 To show capabilities: 
   tv_grab_fr --capabilities
 To show version:  
   tv_grab_fr --version
 Help: 
   tv_grab_fr --help

=head1 DESCRIPTION

Output TV listings for several channels available in France (Hertzian,
Cable/satellite, Canal+ Sat, TPS).  The data comes from
telepoche.guidetele.com.  The default is to grab as many days as possible
(currently 9) from the current day onwards.

B<--configure> Grab channels information from the website and ask for
channel type and names to grab.

B<--configure-more-channels> Use this option to create AUTRES CHAINES list.
This allows grabbing listings for some channels that are not listed  in the 
bouquets on the website. For more informations and a list of channels see 
L<http://www.lahiette.com/biboobox/forum/viewtopic.php?t=1842>.

B<--list-channels> List available channels for chosen bouquets.

B<--config-file FILE> Use FILE as config file instead of the default config 
file. This allow to have different config files for i.e. different apps.

B<--gui OPTION> Use this option to enable a graphical interface to be used.
OPTION may be 'Tk', or left blank for the best available choice.
Additional allowed values of OPTION are 'Term' for normal terminal output
(default) and 'TermNoProgressBar' to disable the use of Term::ProgressBar.

B<--output FILE> Write to FILE rather than standard output.

B<--days N> Grab N days starting from today, rather than as many as
possible.

B<--offset N> Start grabbing N days from today, rather than starting
today.  N may be negative. Due to the website organization, N cannot
be inferior to -1.

B<--quiet> Suppress the progress messages normally written to standard
error.

B<--debug> Provide additional debugging message during processing.

B<--capabilities> Show which capabilities the grabber supports. For more
information, see L<http://wiki.xmltv.org/index.php/XmltvCapabilities>.

B<--version> Show the version of the grabber.

B<--help> Print a help message and exit.

=head1 SEE ALSO

L<xmltv(5)>

=head1 AUTHOR

The original author was Sylvain Fabre, centraladmin at lahiette dot com
with patches from:
  - Francois Gouget, fgouget at free dot fr
  - Niel Markwick, nielm at bigfoot dot com
  - Zubrick, zubrick at number6 dot ch
  - Nick Morrott, knowledgejunkie at gmail dot com

The application is currently maintained by Nick Morrott (knowledgejunkie at gmail dot com).

=cut

# TODO / FIXME
# ============
#
# - perhaps we should internationalize messages and docs?
#
# - try to detect the language based on the country so we can make use of the
#   set the VO, VF and original title markers
#
# - due to the layout of the grid program data, a given show can appear more
#   than once, first without an icon and then with the icon. But in such a
#   case we miss the icon
#
# - investigate how to better handle France 5 and Arte as they share a single
#   channel for over-the-air broadcasts, but each have their own channel on
#   cable, satellite and ADSL. See the thread at:
#   http://sourceforge.net/mailarchive/message.php?msg_id=15181920
#
# - Add caching via HTTP::Cache::Transparent

use XMLTV::Usage <<END
$0: get French television listings in XMLTV format
To configure: tv_grab_fr --configure [--config-file FILE]
To configure AUTRES CHAINES list: tv_grab_fr --configure-more-channels
To grab listings: tv_grab_fr [--config-file FILE] [--output FILE] [--days N]
 [--offset N] [--quiet] [--debug]
To list available channels: tv_grab_fr --list-channels
To show capabilities: tv_grab_fr --capabilities
To show version: tv_grab_fr --version
Help: tv_grab_fr --help
END
  ;

use warnings;
use strict;
use XMLTV::Version '$Id: tv_grab_fr,v 1.97 2011/06/13 07:37:55 dekarl Exp $ ';
use XMLTV::Capabilities qw/baseline manualconfig cache/;
use XMLTV::Description 'France';
use XMLTV;
use XMLTV::Ask;
use XMLTV::ProgressBar;
use XMLTV::Mode;
use XMLTV::Config_file;
use XMLTV::DST;
use XMLTV::Get_nice qw(get_nice); 
$XMLTV::Get_nice::ua->parse_head(0);
$XMLTV::Get_nice::FailOnError = 0;

use XMLTV::Memoize;
XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux');

use Date::Manip; Date_Init("TZ=UTC");
use Encode qw(decode_utf8 encode_utf8);
use Getopt::Long;
use HTML::Entities;
use HTML::TreeBuilder;
use IO::File;
use POSIX qw(strftime);

#***************************************************************************
# Main declarations
#***************************************************************************
my $ROOT_URL                = 'http://telepoche.guidetele.com';
my $GRID_FOR_CHANNEL        = "$ROOT_URL/programmes-tv/";
my $GRID_BY_CHANNEL_PER_DAY = "$ROOT_URL/programmes-tv/chaine/";

my $ENCODING = 'utf-8';
my $LANG = "fr";
my $MAX_RETRY = 5;

# Grid id defined by the website according to channel types (needed to build the URL)
my %GridType = ( "HERTZIENNE"   => "hertziennes",
                 "TNT"          => "tnt",
                 "CABLE/SAT"    => "cable_sat",
                 "ETRANGERES"   => "chaines_etrangeres",
                 "CANAL SAT"    => "canalsat",
                 "FILMS"        => "films",
                 "SPORT"        => "sport",
                 "MUSIQUE"      => "musique",
                 "SERIES"       => "feuilleton",
                 "JEUNESSE"     => "jeunesse",
                 "DOCU/CULTURE" => "documentaire",
                 "FREEBOX"      => "freebox",
               );

#***************************************************************************
# Global variables allocation according to options
#***************************************************************************

my ( $opt_config_file, $opt_days, $opt_offset, $opt_output,
     $opt_gui, $opt_quiet, $opt_list_channels, $opt_configure,
     $opt_morechannels, $opt_help, $opt_debug, );

# Default to non-quiet, non-debug mode
$opt_quiet  = 0;
$opt_debug  = 0;

# The website is able to provide up to nine days of listings
my $default_opt_days = 9;

# Default to STDOUT
$opt_output = '-';

GetOptions( 'days=i'    => \$opt_days,
            'help'      => \$opt_help,
            'output=s'  => \$opt_output,
            'offset=i'  => \$opt_offset,
            'quiet'     => \$opt_quiet,
            'configure' => \$opt_configure,
            'gui:s'     => \$opt_gui,
            'debug'     => \$opt_debug,
            'config-file=s' => \$opt_config_file,
            'list-channels' => \$opt_list_channels,
            'configure-more-channels' => \$opt_morechannels,
    )
    or usage(0);

#***************************************************************************
# Options processing, warnings, checks and default parameters
#***************************************************************************
die 'Number of days must not be negative'  if (defined $opt_days && $opt_days < 0);
die 'Cannot get more than one day before current day' if (defined $opt_offset && $opt_offset < -1);
die 'Error: You cannot specify --quiet with --debug, exiting' if ($opt_quiet && $opt_debug);
    
usage(1) if $opt_help;

XMLTV::Ask::init($opt_gui);

# The days/offset options can be used, but we default them if not set.
$opt_offset = 0 if not defined $opt_offset;
$opt_days = $default_opt_days if not defined $opt_days;

if ( (($opt_offset + $opt_days) > $default_opt_days) or ($opt_offset > $default_opt_days) ) {
    $opt_days = $default_opt_days - $opt_offset;
    if ($opt_days < 0) {
        $opt_offset = 0;
        $opt_days = $default_opt_days;
    }
    say <<END
The website does not handle more than $default_opt_days days.
So the grabber is now configure with --offset $opt_offset --days $opt_days
END
;
}

#***************************************************************************
# Last init before doing real work
#***************************************************************************

# Detect if we are in configure mode
my $mode = XMLTV::Mode::mode('grab', # default
                        $opt_configure => 'configure',
                        $opt_list_channels => 'list-channels',
                        $opt_morechannels  => 'confmorechannels' );

# File that stores which channels to download
my $config_file = XMLTV::Config_file::filename($opt_config_file, 'tv_grab_fr', $opt_quiet);

#***************************************************************************
# Subroutines
#***************************************************************************
sub process_channel_grid_page( $$$$ );
sub process_program_table( $$$$ );
sub process_prog_details_page( $$$ );
sub get_channels( $ );
sub get_manual_config_filename();
sub get_manual_channels();
sub get_manual_channel_icon( $ );
sub debug_print( $ );
sub trim( $ );
sub tidy_html($ );
sub parse_name_list( $ );
sub get_tree( $ );
sub get_country_code( $ );
sub get_date_ymd();
sub get_date_offset_ymd( $ );

#***************************************************************************
# Configure mode
#***************************************************************************
if ($mode eq 'configure') {
    XMLTV::Config_file::check_no_overwrite($config_file);
    open(CONF, ">$config_file") or die "Cannot write to $config_file: $!";

    # Dynamic upgrade of GridType if a file OTHERCHANNELS exists
    my $otherchan_file = get_manual_config_filename();
    if ( (-e $otherchan_file) && ((-s $otherchan_file) > 0) ) {
      $GridType{"AUTRES CHAINES"} = "OTHERCHANNELS";
    }

    # Get a list of available channels, according to the grid type
    my @gts = sort keys %GridType;
    my @gtnames = map { $GridType{$_} } @gts;
    my @gtqs = map { "Get channels type : $_?" } @gts;
    my @gtwant = ask_many_boolean(1, @gtqs);

    my $bar = new XMLTV::ProgressBar('getting channel lists',
                                    scalar grep { $_ } @gtwant)
                    if not $opt_quiet;
    my %channels_for;
    foreach my $i (0 .. $#gts) {
        my ($gt, $gtw, $gtname) = ($gts[$i], $gtwant[$i], $gtnames[$i]);
        next if not $gtw;
        my %channels = get_channels( $gtname );
        die 'No channels could be found' if not %channels;
        $channels_for{$gt} = \%channels;
        update $bar if not $opt_quiet;
    }
    $bar->finish() if not $opt_quiet;

    my %asked;
    foreach (@gts) {
        my $gtw = shift @gtwant;
        my $gtname = shift @gtnames;
        if ($gtw) {
            my %channels = %{$channels_for{$_}};
            say "Channels for $_";

            # Ask about each channel (unless already asked).
            my @chs = grep { not $asked{$_}++ } sort keys %channels;
            my @names = map { $channels{$_}{name} } @chs;
            my @qs = map { "add channel $_?" } @names;
            my @want = ask_many_boolean(1, @qs);
            foreach (@chs) {
                my $w = shift @want;
                warn("cannot read input, stopping channel questions"), last if not defined $w;
                # Print a config line, but comment it out if channel not wanted.
                print CONF '#' if not $w;
                print CONF "channel $_ $channels{$_}{name};$channels{$_}{icon}\n";
            }
        }
    }
    close CONF or warn "cannot close $config_file: $!";
    say("Finished configuration.");
    exit();
}

#***************************************************************************
# "Configure more channels" mode
#***************************************************************************
sub display_otherchannels_list( \% ) {
    my %chlist = %{(shift)};

    say ">>>>>> Current list <<<<<<";
    foreach my $chid (keys %chlist) {
        say "Channel ID: ". $chid." - Name: " . $chlist{$chid}{name} .
            " - Icon: ". $chlist{$chid}{icon};
    }
    say ">>>>>> List end <<<<<<";
}

if ($mode eq 'confmorechannels') {
    my $input_file_notempty = 0;
    my %morechannels = get_manual_channels();
    display_otherchannels_list(%morechannels);
    if ( (scalar keys  %morechannels) > 0 ) {
        $input_file_notempty= 1;
    } 
    my $choice = "";
    my ($chid, $chname, $chicon);

    while ( !($choice eq "exit")) {
        $choice = ask_choice( "Select command to configure OTHERCHANNELS", "add", ("add", "remove", "view list", "save&exit", "exit") );
        my $exit = 0;
        if ($choice eq "add" ) {
            while ($exit == 0) {
                $chid = ask('Enter channel ID : ');
                $chname = ask('Enter channel name : ');
                if ( !($chid =~ /^[0-9]*$/) ) {
                    say ("Enter a numeric value for channel id");
                } else {
                    if ( $chname eq "" ) {
                        say ("Enter a string for the name of the channel");
                    } else {
                        $exit = 1;
                    }
                }
            }
        say("Testing channel $chid - $chname ...");
        my $chicon = get_manual_channel_icon( $chid );
        $morechannels{$chid} = {'name'=>$chname, 'icon'=>$chicon};
        }
        if ($choice eq "remove" ) {
            display_otherchannels_list(%morechannels);
            my $chid = ask('Enter the channel id to remove it (see list above): ');
            if ( defined $morechannels{$chid} ) {
                $chname = $morechannels{$chid}{name};
                delete $morechannels{$chid};
                say ("Channel $chname removed");
            } else {
                say("Channel $chid does not exist in the list");  
            }
        }
        if ($choice eq "view list" ) {
            display_otherchannels_list(%morechannels);
        }
        if ($choice eq "save&exit") {
            my $morechannels_file = get_manual_config_filename();
            # Then write the file
            if ( (scalar keys  %morechannels) > 0 ) {
                open(CONFMORE, ">$morechannels_file") or die "Cannot write to $morechannels_file: $!";
                foreach $chid (keys %morechannels) {
                    if (!( $morechannels{$chid}{name} eq 'DELETED' )) {
                        print CONFMORE "channel $chid $morechannels{$chid}{name};$morechannels{$chid}{icon}\n";
                    }
                }
                close CONFMORE or warn "cannot close $morechannels_file: $!";
                display_otherchannels_list(%morechannels);
                say ('Channel list saved. Launch now a --configure mode to add them into the legacy config');
            } else {
                unlink ($morechannels_file);
                say ('No channels to be configure, file deleted.');
            }
            $choice = "exit";
        }
    } 
    say("Finished configuration for OTHERCHANNELS.");
    exit();
}

#***************************************************************************
# Check mode checking and get configuration file
#***************************************************************************
die if $mode ne 'grab' and $mode ne 'list-channels';

#***************************************************************************
# Prepare the XMLTV writer object
#***************************************************************************
my %w_args;
if (defined $opt_output) {
    my $fh = new IO::File(">$opt_output");
    die "cannot write to $opt_output: $!" if not defined $fh;
    $w_args{OUTPUT} = $fh;
}

$w_args{encoding} = "$ENCODING";
$w_args{days}     = "$opt_days";
$w_args{offset}   = "$opt_offset";
$w_args{cutoff}   = "000000";

my $writer = new XMLTV::Writer(%w_args);
$writer->start
  ({ 'source-info-url'     => "$ROOT_URL/",
     'source-data-url'     => "$ROOT_URL/",
     'generator-info-name' => 'XMLTV',
     'generator-info-url'  => 'http://xmltv.org/',
   });

#***************************************************************************
# List channels only case
#***************************************************************************
if ($mode eq 'list-channels') {
    # Get a list of available channels, according to the grid type
    my @gts = sort keys %GridType;
    my @gtnames = map { $GridType{$_} } @gts;
    my @gtqs = map { "List channels for grid : $_?" } @gts;
    my @gtwant = ask_many_boolean(1, @gtqs);

    my %seen;
    foreach (@gts) {
        my $gtw = shift @gtwant;
        my $gtname = shift @gtnames;
        if ($gtw) {
            say  "Now getting grid : $_ \n";
            my %channels = get_channels( $gtname );
            die 'no channels could be found' if (scalar(keys(%channels)) == 0);
            foreach my $ch_did (sort(keys %channels)) {
                my $ch_xid = "C".$ch_did.".telepoche.com";
                $writer->write_channel({ id => $ch_xid,
                                         'display-name' => [[encode_utf8( $channels{$ch_did}{name} )]],
                                         'icon' => [{src=>$channels{$ch_did}{icon}}] })
                unless $seen{$ch_xid}++;
            }
       }
     }
     $writer->end();
     exit();
}

#***************************************************************************
# Now the real grabbing work
#***************************************************************************
die if $mode ne 'grab';

#***************************************************************************
# Build the working list of channel name/channel id
#***************************************************************************
my @config_lines;
@config_lines = XMLTV::Config_file::read_lines($config_file);

my (%channels, $chicon, $chid, $chname);
my $line_num = 1;
foreach (@config_lines) {
    ++ $line_num;
    next if not defined;

    # Here we store the channel name with the ID in the config file, as 
    # XMLTV id = Website ID
    if (/^channel:?\s+(\S+)\s+([^\#]+);([^\#]+)/) {
        $chid = $1;
        $chname = $2;
        $chicon = $3;
        $chname =~ s/\s*$//;
        $channels{$line_num} = {'chid'=>$chid, 'name'=>$chname, 'icon'=>$chicon};
    } else {
        warn "$config_file:$line_num: bad line $_\n";
    }
}

#***************************************************************************
# Now process the days by getting the main grids.
#***************************************************************************
my @to_get;
warn "No working channels configured, so no listings\n" if not %channels;
my $script_duration = time();

# The website stores channel information by hour area for a whole week !
my $ind;
debug_print("\nCreating list of pages to grab based on configured channels...");
foreach $ind (sort { $a <=> $b } keys %channels) {
    my $chid = $channels{$ind}{chid};
    my $xmltvid = "C".$chid.".telepoche.com";
    my $chname = $channels{$ind}{name};
    my $url;
    my $i;
    my $dayoff;
    $writer->write_channel({ id => $xmltvid, 
                             'display-name' => [[encode_utf8( $chname )]], 
                             'icon' => [{src=>$channels{$ind}{icon}}] });
    debug_print("\n  <channel>: xmltvid=$xmltvid, name=$chname, icon=$channels{$ind}{icon}");
    debug_print("  Creating list of pages for " . $chname);
    for ($i = $opt_offset; $i < $opt_offset+$opt_days; $i++ ) {
        $dayoff = get_date_offset_ymd($i);
        debug_print( "    offset i=$i, date=$dayoff");
        $url = $GRID_BY_CHANNEL_PER_DAY . "$chid/telepoche/soir/$dayoff";
        # create list of urls to process based on configured channels and days/offset
        push @to_get, [ $url, $chid, $i ];
        debug_print( "    adding: url=$url, chid=$chid, i=$i" );
    }
}    

my $bar = new XMLTV::ProgressBar('getting listings', scalar @to_get) if not $opt_quiet;

# Hash to track processed programmes by programme ID
my %processed;
  
debug_print("\nProcessing list of pages to grab...\n");
foreach (@to_get) {
    my ($url, $chid, $slot) = @$_;
    # now process each page of listings data (1 page per channel per day)
    process_channel_grid_page($writer, $chid, $url, $slot);
    update $bar if not $opt_quiet;
}

$writer->end();
$bar->finish() if not $opt_quiet;

# Print the duration
$script_duration = time() - $script_duration;
print STDERR "Grabber process finished in " . $script_duration . " seconds.\n" if not $opt_quiet;

#***************************************************************************
# Subroutines
#***************************************************************************

# Process a page of listings for a particular channel on a particular day,
# based on the given url. The page is designed as a grid of program cells,
# arranged in 6 rows (tables) of 4hrs each (4h-8h, 8h-12h, ...., 0h-4h). 
# Each of these tables is processed in via process_program_table() when 
# identified to extract the programmes contained therein
#
sub process_channel_grid_page( $$$$ ) {
    my ($writer, $chid, $url, $slot) = @_;
    debug_print("process_channel_grid_page(): chid=$chid, url=$url, slot=$slot\n");

    # Get the current page
    my $t = get_tree($url);
    if (not defined $t) {
        debug_print("*** Error: Could not get tree for '" . $url . "' ***");
        return;
    }
    
    # Reset some working variables
    my ($day, $month, $year) = 0;

    # Determine the date of the current page's listings (it should agree with that
    # in the page's url
    if ( my $div_channav = $t->look_down('_tag', 'div', 'id', 'channel_navigation') ) {
        if ( my $logo_text = $div_channav->look_down('_tag', 'span', 'id', 'logo_text') ) {
            my $text_date = $logo_text->as_text();
            $text_date =~ /^Les programmes de la journ.*e\s+du\s+(\d+)\/(\d+)\/(\d+)/;
            $day = $1; $month = $2; $year = $3;
            debug_print("  ###############################################################");
            debug_print("  processing listings for: ".$year."/".$month."/".$day);
            debug_print("  ###############################################################\n");
        }
    }

    my $date = DateCalc($month."/".$day."/".$year, "+0 days");
    my $dateindex = UnixDate($date, "%Y%m%d");
    # process all program cells in turn
    foreach my $tbl_prog ($t->look_down('_tag', 'table', 'class', 'programs')) {
        # Skip table which do not contain program (class 'program')
        next if($tbl_prog->look_down('_tag', 'td', 'class', 'program'));
        debug_print("  processing listings table '" . $tbl_prog->as_text() . "'");
        process_program_table( $writer, $chid, $tbl_prog, $dateindex);
    }
    
    $t->delete(); undef $t;
}

# Process a table of programme cells from the channel grid page. Each cell
# contains basic information which is extracted and a link to a dedicated 
# page of more detailed information for the programme which is processed
# via process_prog_details_page()
#
sub process_program_table( $$$$ ) {
    my ($writer, $chid, $myrow, $dateindex) = @_;
    debug_print("      process_program_table: chid=$chid, dateindex=$dateindex");
    
    # Each show is described in a table cell with one of several class values:
    #
    # - 'program ' - a regular program that has title displayed
    #
    # - 'program previous' - a duplicate program cell that is present because
    #   the programme starts/stops in two different 4hr blocks in the grid.
    #   Programmes starting the previous day are flagged this way
    #
    # - 'program short ' - a short programme (<20min) that is displayed in
    #   the grid as a small icon (no title shown)
    #
    foreach my $show ($myrow->look_down('_tag', 'td', 'class', qr/program /) ) {

        # - Each cell usually contains a link to a detailed description (class = details)
        # - If the cell is wide enough, the link is appplied to the programme title text
        #   and the start time of the programme is also shown (class = hour)
        # - If the cell is very narrow (<20min), no text is displayed and 
        #   instead a small icon is displayed
        # - An onmouseover attribute for the link contains useful program details that are
        #   passed to a popup function on the page
        # - Programme icons may also be displayed - these are also linked to the
        #   detailed description for the programme
        # - If no information is available for part of a 4hr block, the cell can contain 
        #   a cell with the text 'Pas de programme'
        # - If there are no programmes at all in the 4hr block, there will be no
        #   programme cells
        #
        my $link = $show->look_down('_tag', 'a', 'class', 'details') ||
                   $show->look_down('_tag', 'a', 'onmouseover', qr/showmenu/);
        next if (!$link);

        # Each showing has a unique ID present in the url. Extract this and 
        # use it to avoid reprocessing duplicate cells
        my ($prog_url, $prog_id);
        if ( $link->attr('class') eq 'details' ) {
            $prog_url = $link->attr('href');
                debug_print("        Programme URL: " . $prog_url);
            # programme ID are usuallly 8 digits long for valid programmes.
            # Blocks of time containing no programmes have an ID ending with 'Z'
            if ( trim($prog_url) =~ /emission\/(\d+Z?)\// ) {
                $prog_id = $1;
                debug_print("        Programme ID : " . $1);
            }
        }
        if ($processed{$prog_id}) {
            debug_print("        *** Programme already processed, skipping ***\n\n");
            next;
        }

        # The trimmed onmouseover attribute will contain something like:
        # showmenu('23h25-00h20<br />&nbsp;Série/Feuilleton, Thriller','Dexter - Deux en un')
	# However some contain badly escaped double-quotes which causes this
	# string to be truncated. That should really only happen in the title.
        my $line = trim($link->attr('onmouseover'));
        #debug_print("        RAWDATA: '" . $line . "'");
        $line =~ /showmenu\(\'(.*)\',\'((?:[^\\\']|\\.)*)(?:\'|$)/;
        my $tgdata = trim($1); # time/genre data
        debug_print("        Time/genre info: '" . $tgdata . "'");
        my $title = trim($2);  # title data
        $title =~ s%\\\'%\'%g; # unescape the quotes
        $title =~ s%\\\\%\\%g; # and backslashes
        debug_print("        Title info (original): '" . $title . "'");
        # Ignore empty programming cells
        next if ( ($title =~ /Fin de.* programme.*/) or ($title =~ /Pas de programme/) );
        my ($hours, $genre) = split (/<br \/>/, $tgdata);
        next if ( !$hours );
        # At this point, $title contains title and subtitle (if any), separated by a '-'

        # Process start/stop times given as HHhMM
        my ($starthour, $endhour)  = split("-", $hours);
        $starthour =~ s/h//g
          or die "Cannot detect start hour from website : $starthour \n";
        $endhour   =~ s/h//g
          or die "Cannot detect end hour from website : $endhour \n";
        # Process the start/stop dates
        my $start = $dateindex.$starthour."00";
        my $stop  = $dateindex.$endhour."00";
        debug_print("        Initial start date/time: '" . $start . "'");
        debug_print("        Initial stop  date/time: '" . $stop  . "'");
        
        # Initial check to see if start time is later than stop time. This
        # can indicate that a programme is straddling midnight.
        if ( Date_Cmp( ParseDateString($start), ParseDateString($stop) ) > 0 ) {
            $stop = &UnixDate(&DateCalc($stop, "+1 day"), "%Y%m%d%H%M%S");
            die 'could not add one day to stop time' if not $stop;
        }
        
        # Now set the proper timezone (WT/ST) according to current date
        die if not $start; die if not $stop;
        # Do not apply utc_offset until we are happy timings are correct
        
        debug_print("        Creating programme hash for '" . $title . " / " . $start . " " . $stop);
        my %prog = (channel => "C".$chid.".telepoche.com",
                    title   => [ [ encode_utf8( trim($title) ) ] ], # lang unknown
                    start   => $start,
                    stop    => $stop,
                   );
        
        # Process the title sub-title in all cases from the popup title
        # The text for the <a> tag contains the title without the sub-title 
        # so we can use that to separate the two. However the text for the <a> 
        # tag may have been truncated so it fits the slot on the page. Also 
        # some titles may contain a ' - '. Still the heuristic works very 
        # well. In addition, sometimes, the title is not present as text, 
        # only with icons notably in case of short programs.
        #
        my $subtitle;
        my $text = $link->as_text();
        if ( $text eq "") {
            if ( $title =~ m/^(.*) - (.*)$/ ) {
                $prog{'title'}     = [ [ encode_utf8( trim($1) ) ] ];
                $prog{'sub-title'} = [ [ encode_utf8( trim($2) ) ] ];
            } 
        } else {
            if ($text =~ s/\.\.\.$//) {
                if ($title =~ s/^\Q$text\E([^-]+)\s+-\s+//) {
                    $prog{'title'}     = [ [ encode_utf8( trim($text).trim($1) ) ] ];
                    $prog{'sub-title'} = [ [ encode_utf8( trim($title) ) ] ];
                }
            }
            elsif ($title =~ s/^\Q$text\E\s+-\s+//) {
                $prog{'title'}     = [ [ encode_utf8( trim($text) ) ] ];
                $prog{'sub-title'} = [ [ encode_utf8( trim($title) ) ] ];
            }
        }

        # Process the genre (and sub-genre if present)
        if  (defined $genre ) {
            my $subgenre;
            ($genre, $subgenre) = split(",", tidy_html($genre) );
            if (defined $genre && $genre =~ /\w+/) { 
                $genre = trim($genre);
                debug_print("        Programme genre:     '" . $genre . "'");

                if (defined $subgenre && $subgenre =~ /\w+/) {
                    $subgenre = trim($subgenre);
                    debug_print("        Programme sub-genre: '" . $subgenre . "'");

                    $prog{category} = [ [ encode_utf8( lc($genre) ), $LANG ], 
                                        [ encode_utf8( lc($subgenre) ), $LANG ] ];
                } 
                else {
                    $prog{category} = [ [ encode_utf8( lc($genre) ), $LANG ] ];
                }
            }
        }

        # Programme thumbnail image if present
        my $img = $show->look_down('_tag', 'img', 'src', qr%/medias/visuels/v/%);
        if ($img) {
            my $url = trim($img->attr('src'));
            # Get the bigger icon icon
            $url =~ s%/visuels/v/%/visuels/i/%;
            push @{$prog{icon}}, {src => $url};
            debug_print("        Found programme icon: '" . $url . "'");
        }

        # Get more detailed program information (description, review, 
        # original title, etc) from programme details page. We can also 
        # use this data to check dates/times etc.
        if ( $link->attr('class') eq 'details' ) {
            process_prog_details_page( $prog_url, \%prog, $title );
        }

        # Check that the start time of the programme is during today
        my $earliest_start = $dateindex.'000000';
        my $latest_start = &UnixDate(&DateCalc($earliest_start, "+1 day"), "%Y%m%d%H%M%S");
        if ( (Date_Cmp( $prog{start}, $earliest_start) >= 0) and (Date_Cmp( $prog{start}, $latest_start) < 0) ) {
            debug_print("        Programme starts during today, will output");
        }
        else {
            debug_print("        Programme does not start during today, skipping\n\n");
            next;
        }

        # Add timezone information to start/stop times
        $prog{start} = utc_offset( $prog{start}, "+0100");
        $prog{stop}  = utc_offset( $prog{stop} , "+0100");

        # Finally, write the programme
        $writer->write_programme(\%prog);
        # Add ID to processed list
        $processed{$prog_id} = $prog_id;
        debug_print("        Writing programme '" . ${$prog{title}[0]}[0] . " (" . $prog_id . ")'\n\n");
    }
}

# Process a page of detailed programme information for a given url and updates
# the given programme hash with any information extracted from the page
#
sub process_prog_details_page( $$$ ) {
    my $url   = shift;
    my $prog  = shift;
    my $title = shift;

    # unused but data is put into these vars for possible future use
    my $fic_channel;
    # store parsed start time/date/duration data
    my ($p_dur_hr, $p_dur_min) = 0;
    my ($p_hr, $p_min) = 0;
    my ($p_day, $p_month, $p_year) = 0;

    my $fic_url = $ROOT_URL . $url;
    debug_print("\n    process_prog_details_page(): " . $fic_url);
    my $tfic = get_tree($fic_url);
    if (not defined $tfic) {
        debug_print("*** Error: Could not get tree for '" . $fic_url . "' ***");
        return;
    }

    # Get the details for the programme
    if ( my $tdetails = $tfic->look_down('_tag', 'table', 'width', '300', 'class', 'bord') ) {
        debug_print("      Located main programme details table");

        # Process cells in first table - 4 cells: start time, channel, date, duration
        if (my $table_timings = $tdetails->look_down('_tag', 'table', 'class', 'dates') ) {
            debug_print("        Located programme timings table");
            #debug_print("          RAW DATA: '" . $table_timings->as_text() . "'");
            my @td_timings = $table_timings->look_down('_tag', 'td', 'align', 'center');
            #last if $#td_timings != 4;
            if (trim($td_timings[0]->as_text()) =~ /(\d\d)h(\d\d)/) { # e.g. 16h35
                $p_hr = $1;
                $p_min = $2;
                debug_print("          Programme start time: '" . $p_hr.$p_min . "'");
            }
            $fic_channel   = trim($td_timings[1]->as_text()); # e.g. TF1
            debug_print("          Programme channel: '" . $fic_channel . "'");

            # Start to build a starttime to compare to the starttime parsed from the
            # program cell
            if (trim($td_timings[2]->as_text()) =~ /(\d\d)\/(\d\d)\/(\d{4})/) { # e.g. 15/02/2010
                $p_day = $1;
                $p_month = $2;
                $p_year = $3;
                debug_print("          Programme start date: '".$p_day."/".$p_month."/".$p_year."'");
            }   

            if (trim($td_timings[3]->as_text()) =~ /^Dur.*e\s*:\s*(?:(\d{1,2})h)?\s*(\d{1,2})min/) {
                $p_dur_hr = $1 if defined $1;
                $p_dur_min = $2;
                debug_print("          Programme duration: '".$p_dur_hr."h".$p_dur_min."min'");
            }
        }

        # Now we compare the start/stop timings in $prog with those calculated from
        # the information given on the details page. We can adjust any timings found
        # to be different
        my $curr_start = ${$prog}{start};
        my $curr_stop  = ${$prog}{stop};
        my $details_start = $p_year.$p_month.$p_day.$p_hr.$p_min.'00';
        my $details_stop  = &UnixDate(&DateCalc($details_start, "+$p_dur_hr hours $p_dur_min minutes"), "%Y%m%d%H%M%S");

        if ( (Date_Cmp( $curr_start, $details_start ) == 0) and (Date_Cmp( $curr_stop, $details_stop ) == 0) ) {
            debug_print("          Programme start/stop times agree on grid and details pages, no adjustment required");
        }
        else {
            ${$prog}{start} = $details_start;
            ${$prog}{stop}  = $details_stop;
            debug_print("          Programme start times differ on grid and details pages, adjustment required");
            debug_print("            Channel grid: start = " . $curr_start . ", stop = " . $curr_stop);
            debug_print("            Details page: start = " . $details_start . ", stop = " . $details_stop);
            debug_print("              Will use start/stop times from details page");
        }

        debug_print("");

        # Process cells in second table - 5 cells: date, ?, ?, rating, ?
        foreach my $td ($tdetails->look_down('_tag', 'td', 'align', 'center', 'class', 'blc12') ) {

            # Programme rating
            if (my $img = $td->look_down('_tag', 'img')) {

                debug_print("        Located a likely ratings image");
                my $text = $img->attr('alt');
                debug_print("          Programme rating description: '" . $text . "'");
                my $rating;
                
                if ($text =~ /interdit - 10 ans/i) {
                    $rating = "-10";
                } elsif ($text =~ /interdit - 12 ans/i) {
                    $rating = "-12";
                } elsif ($text =~ /interdit - 16 ans/i) {
                    $rating = "-16";
                } elsif ($text =~ /interdit - 18 ans/i) {
                    $rating = "-18";
                } else {
                    debug_print("          Unknown programme rating: '" . $text . "'");
                }
                if ($rating) {
                    my $icon = trim($ROOT_URL . $img->attr('src'));
                    push @{$prog->{rating}}, [ $rating, "CSA", [ {src => $icon} ] ];
                    debug_print("          Programme rating: '" . $rating . "' (icon = " . $icon . ")");
                }
            }
            # Production year
            if ($td->as_text() =~ /^(\d{4})$/i) {
                $prog->{date} = $1;
                debug_print("          Programme year: '" . $1 . "'");
            }
        }
        debug_print("");
        
        # Process content in third table - programme icon, director, writer 
        if ( my $table_cat = $tdetails->look_down('_tag', 'table', 'width', '298', 'cellpadding', '4') ) {
            debug_print("        Located crew/programme icon table");
            #debug_print("          RAW DATA: '" . $table_cat->as_text() . "'");

            # Programme icon is handled in process_program_table()

            # Original title/country/video/audio information
            if (my $td_avinfo = $table_cat->look_down('_tag', 'td', 'class', 'bleuf11r')) {
                my $content = $td_avinfo->as_HTML();
                my ($original_title, $original_country);
                foreach my $field ( split ("<br />", $content) ) {
                    $field = trim( tidy_html($field) );
                    if ( $field =~ /^couleur.*/i ) {
                        $prog->{'video'}{colour} = 1;
                        debug_print("          Programme is in colour");
                    } elsif ( $field =~ /^noir.*/i ) {
                        $prog->{'video'}{colour} = 0;
                        debug_print("          Programme is in black and white");
                    } elsif ( $field =~ /^16\/9/ ) {
                        $prog->{'video'}{aspect} = "16:9";
                        debug_print("          Programme is in widescreen");
                    } elsif ( $field =~ /^(st.r.o|Dolby)\s+/i) {
                        my $stereo = $1;
                        if ($stereo =~ /^st.r.o$/i) {
                            $prog->{'audio'}{stereo} = "stereo";
                            debug_print("          Programme is in stereo");
                        } elsif ($stereo eq "Dolby") {
                            $prog->{'audio'}{stereo} = "dolby";
                            debug_print("          Programme is in Dolby stereo");
                        }
                    }
                    elsif ( $field =~ /^Titre original\s*:\s*<b>(.*)<\/b>/i) {
                        $original_title = $1;
                    }
                    elsif ( $field =~ /^Nationalit.*\s*:\s*<b>(.*)<\/b>/i) {
                        $original_country = $1;
                    }
                }
                # handle country of origin and original title
                if (defined $original_country) {
                    my ($c_lang, $c_code) = get_country_code($original_country);
                    if (defined $c_code) {
                        debug_print("          Programme originated in '" . $original_country 
                                    . "' (country code '" . $c_code . "')");
                        push @{$prog->{country}}, [ encode_utf8( $original_country ), $LANG ];
                        push @{$prog->{country}}, [ encode_utf8( uc($c_code) ) ];
                    }
                    else {
                        debug_print("          Programme originated in '" . $original_country 
                                    . "' (unknown country code)");
                    }
                    # only output original title if we know the language it is given in
                    if (defined $original_title and defined $c_lang) {
                        push @{$prog->{title}}, [ encode_utf8( $original_title ), $c_lang ];
                        debug_print("          Programme has original title '" . $original_title . "'");
                    }
                }
            }

            # Director/writer/presenter/music etc
            foreach my $crew ($tdetails->look_down('_tag', 'span', 'class', 'bleuf11r')) {
                my $sibling = $crew->left();
                next if (!defined $sibling);
                
                # find matching <span> element
                if (!ref $sibling) {
                    # $sibling is a string, what we want is the previous <span> element
                    $sibling = ($crew->parent()->content_list())[$crew->pindex()-2];
                    next if (!defined $sibling);
                }
                next if ($sibling->attr('_tag') ne "span");
                
                my $job = trim($sibling->as_text());
                my $name = trim($crew->as_text());
                # now check jobs against supported credit types
                if ($job =~ /R.*alisateur/i) {
                    push @{$prog->{credits}{director}}, encode_utf8( $name );
                    debug_print("          Found director: '" . $name . "'");
                } elsif ($job =~ /Pr.*sentation/i) {
                    push @{$prog->{credits}{presenter}}, encode_utf8( $name );
                    debug_print("          Found presenter: '" . $name . "'");
                } elsif ($job =~ /Sc.*nari.*ste/i) {
                    push @{$prog->{credits}{writer}}, encode_utf8( $name );
                    debug_print("          Found writer: '" . $name . "'");
                }
            }
        }
    }

    # Process the cast list
    if ( my $casting = $tfic->look_down('_tag', 'table', 'width', '609', 'cellpadding', '4') ) {
        debug_print("      Located cast details table");
        foreach my $actor ($casting->look_down('_tag', 'td', 'valign', 'top', 'class', 'bleuf11' )) {
            next if ($actor->as_text() eq "Acteur");
            next if ($actor->as_text() =~ /^\s*$/);
            my $role = $actor->right();
            my ($name, $part);
            $name = trim($actor->as_text());
            if (defined $role and ($role->attr('class') eq "bleuf11r")) {
                $part = trim($role->as_text()) if trim($role->as_text()) !~ /\s*/;
            }
            # No support for role in actor element
            if (defined $part) {
                #push @{$prog->{credits}{actor}}, [ encode_utf8($name), { role => encode_utf8($part)} ];
                debug_print("        Found actor: '" . $name . "' ('" .  $part . "')");
            }
            else {
                #push @{$prog->{credits}{actor}}, encode_utf8($name);
                debug_print("        Found actor: '" . $name . "'");
            }
            push @{$prog->{credits}{actor}}, encode_utf8($name);
        }
    }

    # Get summary, description, director, etc
    my ($histoire, $resume, $avis);
    if (my $desc_table = $tfic->look_down('_tag', 'table', 'class', 'bord', 'cellpadding', '4')) {
        debug_print("      Located description details table");
        if (my $desc_td = $desc_table->look_down('_tag', 'td', 'class', 'bleuf11r')) {
            my @descs = split( "<br />", $desc_td->as_HTML() );
            for (my $tmp = 0; $tmp < $#descs; $tmp++) {
                my $desc = trim( tidy_html($descs[$tmp]) );
                if ($desc =~ /Histoire :/i) {
                    $histoire = trim( tidy_html($descs[$tmp+1]) );
                    debug_print("        Programme histoire: '" . $histoire . "'");
                }
                elsif ($desc =~ /R.*sum.* :/i) {
                    $resume = trim( tidy_html($descs[$tmp+1]) );
                    debug_print("        Programme resume: '" . $resume . "'");
                }
                elsif ($desc =~ /Avis :/i) {
                    $avis = trim( tidy_html($descs[$tmp+1]) );
                    debug_print("        Programme review: '" . $avis . "'");
                }
            }
        }
    }

    # If histoire is the same as, or part of the resume, ignore histoire text
    if (defined $histoire and defined $resume) {
        if ($histoire eq $resume) {
            undef $histoire;
            debug_print("        Programme summary and description match, dropping summary");
        }
        elsif ($resume =~ /\Q$histoire\E/i) {
            undef $histoire;
            debug_print("        Programme summary is included in description, dropping summary");
        }
    }
    
    # $resume is the main description, $histoire is a short summary
    foreach ($resume, $histoire) {
        push @{$prog->{desc}}, [ encode_utf8( $_ ), $LANG ] if defined and length;
    }
    # Add $avis to existing description, otherwise add as a new desc
    if (defined $avis and length($avis) ) {
        if ($prog->{desc}) {
            $prog->{desc}->[0]->[0] .= encode_utf8( " Critique : " . $avis );
        } else {
            push @{$prog->{desc}}, [ encode_utf8( $avis ), $LANG ];
        }
    }

    $tfic->delete(); undef $tfic;
}

# Return a hash of available channels for a given bouquet. Channels configured 
# in --configure-more-channels mode are retrieved via get_manual_channels()
#
sub get_channels( $ ) {
    my $bouquet = shift;
    my %bouquet_channels;

    return undef if not defined $bouquet;

    # Other channels mode, return the content of the file
    if ( $bouquet eq 'OTHERCHANNELS' ) {
        return get_manual_channels();
    }

    # Get the current page for the given bouquet
    my $today = get_date_ymd();
    my $url = $GRID_FOR_CHANNEL.'grille/'.$bouquet.'/soiree/'.$today;
    my $t = get_tree($url);
    debug_print("get_channels(): url = '" . $url . "'");

    foreach my $cellTree ( $t->look_down( "_tag", "td", "class", "channel" ) ) {
        my $tag = $cellTree->look_down( "_tag", "a" ); 
        if (defined $tag) {
            my $progurl = $tag->attr('href');
            if ( $progurl =~ /^\/programmes-tv\/chaine\/(\d+)/ ) {
                $chid = $1;
                my $imgCell = $cellTree->look_down( "_tag", "img" );
                my $chicon = trim( $ROOT_URL.$imgCell->attr('src') );
                my $chname = trim( $imgCell->attr('alt') );
                debug_print("  Found channel: id = '" . $chid . "', name = '" . $chname . "'");
                $bouquet_channels{$chid} = {'name' =>  $chname, 'icon' => $chicon };
            }
        }
    }
    $t->delete(); undef $t;
    return %bouquet_channels;
}

# Return a hash of the channels configured manually
#
sub get_manual_channels() {
    my $morechannels_file = get_manual_config_filename();
    my @morechannels_lines;
    if (-e $morechannels_file && ((-s $morechannels_file)>0) ) {
        @morechannels_lines = XMLTV::Config_file::read_lines($morechannels_file);
    } 
    my %morechannels;
    my ($chid, $chname, $chicon);
    my $line_num = 0;

    foreach (@morechannels_lines) {
        next if not defined;

        # store the channel name and icon, keyed against the website ID
        # for the channel
        if (/^channel:?\s+(\S+)\s+([^\#]+);([^\#]+)/) {
            $chid = $1;
            $chname = $2;
            $chicon = $3;
            $chname =~ s/\s*$//;
            $morechannels{$chid} = {'name'=>$chname, 'icon'=>$chicon};
        }
        $line_num++;
    }
    return %morechannels;
}

# Return the full path of the configuration file used to store 
# channels configured manually (--configure-more-channels)
#
sub get_manual_config_filename() { 
    my $home = $ENV{HOME};
    $home = '.' if not defined $home;
    my $conf_dir = "$home/.xmltv";
    (-d $conf_dir) or mkdir($conf_dir, 0777) or die "cannot mkdir $conf_dir: $!";
    return "$conf_dir/OTHERCHANNELS";
}

# Return a channel icon url for a given channel id
# This sub is called from --configure-more-channels mode in
# order to add an icon url to the config file
#
sub get_manual_channel_icon( $ ) {
    my $chid = shift;
    my $today = get_date_ymd();
    my $url = $GRID_BY_CHANNEL_PER_DAY.$chid.'/telepoche/soiree/'.$today;
    print $chid;  
    # Get the current page
    my $t = get_tree($url);
    
    debug_print("get_manual_channel_icon(): url = $url\n");
    # Set by default an EMPTY logo
    my $chicon = "EMPTY";

    foreach my $cellTree ( $t->look_down( "_tag", "img") ) {
        my $chiconsrc = $cellTree->attr('src');
        if ( $chiconsrc =~ /\/medias\/chaines\/(.*)/ ) {
            $chicon = "$ROOT_URL/medias/chaines/".$1;
            debug_print("  Found icon, url = $chicon");
        }
    }
    $t->delete(); undef $t;
    return $chicon;
}

# Prints a debug string when the --debug option is used
#
sub debug_print( $ ) {
    if ($opt_debug) {
        my ($msg) = shift;
        print STDERR $msg . "\n";
    }
}

# Remove leading/trailing whitespace
#
sub trim( $ ) {    
    for (my $s = shift) {
        s/^\s*//;
        s/\s*$//;
        
        return $s;
    }
}

# Replaces specific HTML entities with text replacements, and then 
# decodes any remaining entities in the string
#
sub tidy_html( $ ) {    
    for (my $s = shift) {
        # handle specific entities
        s/&nbsp;/ /g;
        s/\x8c/&OElig;/g;
        s/&OElig;/OE/g;
        s/\x9c/&oelig;/g;
        s/&oelig;/oe/g;
        s/&ldquo;|&rdquo;|&\#8219;|&\#8220;/\"/g;
        s/&lsquo;|&rsquo;|&\#8216;|&\#8217;|&\#8218;/\'/g;
        s/&\#8212;/--/g;
        s/\x85/.../g;
        s/&\#8230;/.../g;
        s/&\#8230;/.../g;
        s/&\#821[0123];/-/g;
        s/\xe2\x80\x99/\'/g;
        s/\x92/\'/g; # turn windows-1252 right single quotation mark into apostrophe
        s/\x80/&#x20ac;/g; # turn windows-1252 euro sign into real euro sign
        s/(&\#[0-9]{4,};)//g;
        # decode remaining entities
        decode_entities($s);
        
        return $s;
    }
}

# The given string is processed to produce a comma-separated list of
# names, which is then returned
#
sub parse_name_list( $ ) {
    for (my $s = shift) {
        $s = trim($s);
        $s = s/\s+(?:avec|et)\s+/, /i;
        
        return split /(?:\s*,)+\s*/, $s;
    }
}

# Returns a TreeBuilder instance for a given url. The url is retrieved
# via get_nice(), decoded into a Perl string, processed to remove HTML
# entities and then parsed into a HTML::TreeBuilder object
#
sub get_tree( $ ) {
    my $url = shift;
    my $content;
    my $nbretry = 0;
    while (not defined($content = get_nice($url))) {
        ++$nbretry;
        return undef if $nbretry == $MAX_RETRY;
        debug_print("*** Retrying URL: '" . $url 
                . "' (attempt " . $nbretry . " of " . $MAX_RETRY . ") ***");
    }
    $content = decode_utf8($content);
    $content = tidy_html($content);
    my $t = new HTML::TreeBuilder;
    $t->parse($content) or die "Cannot parse content of Tree\n";
    $t->eof;
    return $t;
}

# Return a list containing:
#
#   1) the two character language code for a country (given in French)
#      per ISO 639-1 (http://en.wikipedia.org/wiki/ISO_639-1)
#   2) the two character country code for a country (given in French) 
#      per ISO 3166-1 alpha-2 (http://en.wikipedia.org/wiki/ISO_3166-1)
#
sub get_country_code( $ ) {
    my $fr_country = shift;
    return undef if !defined $fr_country;
    for ($fr_country) {
        if    (/France/i)     {return ('fr', 'FR')}
        elsif (/Angleterre/i) {return ('en', 'GB')}
        elsif (/Etats-Unis/i) {return ('en', 'US')}
        elsif (/Canada/i)     {return ('en', 'CA')}
        elsif (/Allemagne/i)  {return ('de', 'DE')}

        # return undef if input unhandled
        else { return undef }
    }
}

# Return today's date as a string in format "YYYY-MM-DD"
#
sub get_date_ymd() {
    return POSIX::strftime( "%Y-%m-%d", localtime() );
}

# Return a date string in format "YYYY-MM-DD", calculated as the given
# number of days from today's date
#
sub get_date_offset_ymd( $ ) {
    my $offset = shift;
    return POSIX::strftime( "%Y-%m-%d", gmtime(time() + 3600 * 24 * $offset) );
}