grid-packaging-tools 3.6.7-1 / usr / sbin / gpt-build

#! /usr/bin/perl


my ($prefix, $globus_prefix, $exec_prefix, $bindir, $sbindir, $libdir);
my ($datarootdir, $datadir, $perlmoduledir, $gptdatadir);
my ($gptexecsharedir, $amdir, $pkg_confdir, $aclocaldir);

if (exists $ENV{GPT_LOCATION})
{
    $prefix = $ENV{GPT_LOCATION};
}
else
{
    $prefix = "/usr";
}

if (exists $ENV{GLOBUS_LOCATION})
{
    $globus_prefix = $ENV{GLOBUS_LOCATION};
}
else
{
    $globus_prefix = "/usr";
}

$exec_prefix = "${prefix}";
$bindir = "${exec_prefix}/bin";
$sbindir = "${exec_prefix}/sbin";
$libdir = "${exec_prefix}/lib";
$datarootdir = "${prefix}/share";
$datadir = "${prefix}/share";
$perlmoduledir = "/usr/share/perl5";
$gptdatadir = "${datadir}/globus/gpt";
$gptexecsharedir = "${datadir}/globus/gpt";
$amdir = "${datadir}/globus/amdir";
$pkg_confdir = "${datadir}/globus/gpt";
$aclocaldir = "${datadir}/globus/aclocal";

unshift(@INC, "${perlmoduledir}");

use strict;
use Getopt::Long;
use File::Find;
use Cwd;
use Config;

#
# Do a perl check for version >= 5.005.
#

if ( ! ( defined eval "require 5.005" ) )
{
    die "GPT requires at least Perl version 5.005";
}

if ( ! ( defined eval "require Grid::GPT::GPTObject" ) )
{
    die("$prefix does not appear to hold a valid GPT installation\n");
}

my $VERSION = "0.01";

system("$sbindir/gpt_version") == 0
         or die "GPT died due to Version mismatch.  Check PATH and GPT_LOCATION\n" ;


require Pod::Usage;

# declare all of the option variables
my $startdir = cwd();
my @srcdirs;
my $builddir;
my $confopts = "";
my $logfile;
my $man = 0;
my $help = 0;
my ($verbose, $flavor_cfg, $srcfile, $static, $force, $fhs,
    $installdir, $installonly, $allflavors, $stdflavors, $noflavor, 
    $version, $update, $logdir, $nosrc, $coresrc, $debug, $ignore_errors);
my %macros;

# process the -option options

GetOptions( 'srcdir=s' => \@srcdirs, 
            'srcfile=s' => \$srcfile, 
            'builddir=s' => \$builddir,
            'installdir=s' => \$installdir,
            'location=s' => \$installdir,
            'coresrc=s' => \$coresrc,
            'flavor-cfg=s' => \$flavor_cfg,
            'static:i' => \$static,
            'all-flavors!' => \$allflavors, 
            'std-flavors!' => \$stdflavors, 
            'logdir:s' => \$logdir,
            'force!' => \$force, 
            'fhs!' => \$fhs, 
            'nosrc!' => \$nosrc,
            'update!' => \$update, 
            'ignore-errors!' => \$ignore_errors, 
            'verbose!' => \$verbose, 
            'debug' => \$debug, 
            'help|?' => \$help,
            'man' => \$man,
            'version' => \$version) 
  or Pod::Usage::pod2usage(0);

Pod::Usage::pod2usage(0) if $help;
Pod::Usage::pod2usage(-verbose => 2) if $man;

require Grid::GPT::GPTIdentity;
Grid::GPT::GPTIdentity::print_gpt_version() if defined $version;

$macros{'CONFIGENV_GPTMACRO'}= "CONFIG_SITE=".$gptexecsharedir."/gpt_noinstaller; export CONFIG_SITE;" if defined $fhs;

require Grid::GPT::PkgMngmt::Build;
require Grid::GPT::FilelistFunctions;
require Grid::GPT::PkgMngmt::Inform;
require Grid::GPT::PkgMngmt::ExpandSource;
require Grid::GPT::PkgMngmt::SetupBuildFlavors;
require Grid::GPT::PackageFactory;
require Grid::GPT::Installation;
require Grid::GPT::Locations;

my $locations = new Grid::GPT::Locations(
                                         installdir => $installdir,
                                         builddir => $builddir,
                                        );


my (@flavors, @tarfiles, @buildobjs, $installed_flavors);

my $currentmacro = "";

# seperate the rest of @ARGV into flavors, macros, and tarballs
for my $f (@ARGV) {

#  print "!$f!\n";

  next if $f !~ m!\S+!;

  my $result = extract_macro($f);

  die "ERROR: $f is not a valid macro name\n" if $result eq "INVALID_MACRO";

  next if $result eq "VALID_MACRO";

  if ($f =~ m!\.tar\.gz!) {
    push @tarfiles, $f;
    next;
  }
  push @flavors, $f;
}

$locations->create_dirs(mode => 'install');

my $flavorsetup = 
  new Grid::GPT::PkgMngmt::SetupBuildFlavors(
                                             all => $allflavors,
                                             std => $stdflavors,
                                             conffile => $flavor_cfg,
                                             locations => $locations,
                                             macros => \%macros,
                                            );



# Make sure that Getopt::Long did not suck up flavors or bundles in 
# options with optional values.

my @opt_ptrs = (\$logdir );

for my $opt ( @opt_ptrs ) {
  next if ! defined $$opt;

  if ($$opt =~ m!\.tar\.gz$!) {
    push @tarfiles, $$opt;
    $$opt = "";
  }

  if ($flavorsetup->check_flavor($$opt) ne "NOT_DEFINED" ) {
    push @flavors, $$opt;
    $$opt = "";
  }

  my $result = extract_macro($$opt);

  die "ERROR: $$opt is not a valid macro name\n" if $result eq "INVALID_MACRO";

  if ($result eq "VALID_MACRO") { #valid macro
    $$opt="";
  }
}


# Assume the current directory is a source directory 
# if no tarballs or -srdir's were given
$srcdirs[0] = $startdir if ! defined $srcdirs[0] and ! @tarfiles;

# Process all of the srcdir's
for my $s (@srcdirs) {
  my $src = new Grid::GPT::PkgMngmt::ExpandSource(srcdir => $s,
                                                  srcfile => $srcfile,
                                                  locations => $locations,
                                       );
  push @buildobjs, $src;
}

# Create a build directory if tarballs are given
$verbose = 1 if defined $logfile;

if( defined($logdir) )
  {
    if( $logdir eq "")
      {
        $logdir = './logs';
      }
    Grid::GPT::FilelistFunctions::mkinstalldir($logdir);
    $logdir = Grid::GPT::FilelistFunctions::abspath($logdir);
  }

my $glogname = defined $logdir ? "$logdir/gpt-build.log" : undef;

my $gptlog = 
  new Grid::GPT::PkgMngmt::Inform(verbose => $verbose, 
                                  log => $glogname, 
                                  debug => $debug, 
                                  name => 'gpt-build',
                                 );

$locations->create_dirs(mode => 'build') if @tarfiles;

# assume each tarball is a bundle.  ExpandSource can sort out the rest.
for my $t (@tarfiles) {
  my $sources = 
    Grid::GPT::PkgMngmt::ExpandSource::open_bundle(file => "$t", 
                                                   locations => $locations,
                                                   log => $gptlog
                                                  );
  Pod::Usage::pod2usage(-verbose => 0, 
                        -exitval => 1,
                        -output =>  \*STDERR,
                        -message => "ERROR: Cannot build $t") 
      if ! defined $sources;

  push @buildobjs, @$sources;
}

# Scan source objects for globus_core
my $coreobj;
my @buildobjswithoutcore;
if (! defined $nosrc) {
  for my $o (@buildobjs) {
    
    # Let expand source figure out where various files and directories are located
    my $result = $o->setup_source(
                                  log => $gptlog,
                                  disable_version_checking => 0
                                 );

    Pod::Usage::pod2usage(-verbose => 0, 
                          -exitval => 1,
                          -output =>  \*STDERR,
                          -message => "ERROR: Cannot build $o->{'srcdir'}") 
        if $result;

    if ($o->{'pkg'}->{'Name'} eq "globus_core") {
      $coreobj = $o if $o->{'pkg'}->{'Name'} eq "globus_core";
    } else {
      push @buildobjswithoutcore,$o;
    }
  }
}

# Install any new flavors

$gptlog->announce("Logging new flavor installs in $logdir/globus_core.log") 
  if defined $logdir;


$flavorsetup->set_installation(log => $gptlog);
$flavorsetup->add_flavors(\@flavors);
$flavorsetup->select_core_src(
                              usercore => $coresrc,
                              coreobj => $coreobj,
                             );

$flavorsetup->verify_flavor_list(
                                 force => $force,
                                 nosrc => $nosrc,
                                );
$flavorsetup->build_core(
                         logdir => $logdir,
                         verbose => $verbose,
                         debug => $debug,
                         macros => \%macros,
                         static => $static,
                        );

exit 0 if defined $nosrc; 

# The grand build loop for all of the build objects
for my $o (@buildobjswithoutcore) {

  my $pkg = $o->{'pkg'};
  my $name = $pkg->Name();

  my $logname = undef;
  $logname = $logdir . "/" . $name . ".log"
    if( defined($logdir) );
  my $log = new Grid::GPT::PkgMngmt::Inform(verbose => $verbose, 
                                            log => $logname,
                                            debug => $debug,
                                            name => 'gpt-build');
  my $filelist_funcs = 
    new Grid::GPT::FilelistFunctions(
                                     locations => $locations, 
                                     log=> $log, 
                                     force=> $force,);
  

  $gptlog->announce("Logging build of $name in $logname\n") 
    if defined $logname;
  #Build dependency check
  $log->announce("CHECKING BUILD DEPENDENCIES FOR $name");

  # Extract the list of dependent packages so we
  # can load Installation with only packages we require.
  my @pkgdirs;

  for my $e ( keys %{$pkg->{'Source_Dependencies'}->{'pkgname-list'}} ) {
     next if ($e =~ /setup/);
     push @pkgdirs, $e;
  }

  my $installation = 
    new Grid::GPT::Installation(
                                locations => $locations,
                                disable_version_checking => 0,
                                log => $log,
                                root_package => $pkg,
                                dep_packages => \@pkgdirs,
                                deps => 1,
                               );

  my $node = $installation->add_package(pkg => $pkg);


  # Get the list of installed flavors
  $installed_flavors = $flavorsetup->installed();

  my @local_flavors;
  @local_flavors = @{ $flavorsetup->list() } if $pkg->{'With_Flavors'} eq 'yes';

  if ($pkg->{'With_Flavors'} eq 'yes' and ! @local_flavors) {
    die "ERROR: At least one flavor needs to be defined for package $name\n";
  }

  my $build = 
    new Grid::GPT::PkgMngmt::Build(
                                   srcobj => $o, 
                                   name => $name, 
                                   locations => $locations, 
                                   verbose => $verbose, 
                                   log => $log,
                                   build_instructions =>
                                   $pkg->{'Build_Instructions'},
                                   flavor_choices =>
                                   $pkg->{'Build_Flavor_Choices'},
                                   macros => \%macros,
                                   static => $static,
                                   installed_flavors => $installed_flavors,
                                   ignore_errors => $ignore_errors,
                                   core => $name eq 'globus_core',
                                  );
  my $forcepkg;

  if (@local_flavors) {
    for my $f (@local_flavors) {

      my $result = check_dependencies($node, 
                         $f, 
                         $static,
                         $installation,
                         $log,
                        );
      if ($result eq 'FAILED') {
        die if ! defined $ignore_errors;
        next;
      }

      my $rebuild =  should_rebuild($node, 
                                    $f, 
                                    $force, 
                                    $update, 
                                    $static, 
                                    $installation,
                                    $log);

      $log->debug("$rebuild");


      if ($rebuild eq 'IGNORE') {
        $log->inform("SKIPPING REBUILD of $f",1);
        next; # next flavor
      }
      $forcepkg++;

      $filelist_funcs->backup_pkgdata(
                                      flavor => $f,
                                      pkg => $pkg,
                                     );
      $result = $build->build($f);

      $filelist_funcs->check_installed_files(name => $name,
                                             flavor => $f,
                                             copy_pgm_files => ! defined $installonly,
                                             static => $static,
                                             pkg => $pkg,
					     force=> $forcepkg,
                                             srcdir => $o->{'topsrcdir'});
      $filelist_funcs->restore_pkgdata(
                                      flavor => $f,
                                      pkg => $pkg,
                                     );
    }

    $filelist_funcs->check_installed_files(name => $name,
                                           flavor => 'noflavor',
                                           static => $static,
					   force=> $forcepkg,
                                           pkg => $pkg,
                                             srcdir => $o->{'topsrcdir'});
      $filelist_funcs->restore_pkgdata(
                                      flavor => 'noflavor',
                                      pkg => $pkg,
                                     );
  } else {

      my $result = check_dependencies($node, 
                                      'noflavor', 
                                      $static,
                                      $installation,
                                      $log);
      if ($result eq 'FAILED') {
        die if ! defined $ignore_errors;
        next;
      }
      my $rebuild =  should_rebuild($node, 
                                    'noflavor', 
                                    $force, 
                                    $update, 
                                    $static, 
                                    $installation,
                                    $log);


    if ($rebuild eq 'IGNORE') {
      $log->inform("SKIPPING REBUILD of noflavor",1);
      next;
    }
    $build->build();
    $filelist_funcs->check_installed_files(name => $name,
                                           pkg => $pkg,
                                           static => $static,
					   force=> $forcepkg,
                                           noflavors => 1,
                                           srcdir => $o->{'topsrcdir'});
  }
}


exit 0;

sub extract_macro {
  my ($f) = @_;
  if ($f =~ m!([^=]+)=(.+)!) {
    my ($name, $value) = ($1, $2);
    return "INVALID_MACRO" if $name !~ m!_GPTMACRO!;
    $macros{$name} = $value;
    return "VALID_MACRO"
  }

  return "NOT_A_MACRO";
}

sub should_rebuild {
  my ($newpkg, $flavor, $force, $update, $static, $installation, $log) = @_;

  my $binpkgs = $installation->query(pkgname => $newpkg->pkgname(), 
                                     flavor => $flavor,
                                    );

  @$binpkgs = grep { $_->pkgtype() ne 'src' } @$binpkgs;

  my $msg="";

  for my $p (@$binpkgs) {
    $msg .= "\t" . $p->label() . "-" . $p->version_label() . "\n";
  }

  $log->debug("replacer: " .  $newpkg->pkgname . "-$flavor-src-" . 
                      $newpkg->version_label()  . 
                      " replacee: \n$msg" .  
                     ($update ? "UPDATE " : " ") .
                     ($static ? "STATIC " : " ") .
                     ($force ? "FORCE " : " ")
                     );

  $log->debug("Testing for installed " . $newpkg->pkgname() . " binary packages");

  return 'REBUILD' if ! @$binpkgs;

  my $oldpkg = $binpkgs->[0];


  # force is defined

  $log->debug("Testing for FORCE");

  return "REBUILD" if defined $force;

  # old is newer than new

  $log->debug("Testing  if installed pkgs are newer");

  return "IGNORE" 
    if $oldpkg->{'depnode'}->{'Version'}->is_newer($newpkg->{'depnode'}->{'Version'})
      and defined $update;

  # new and old are the same flavor and pkgtype and new is newer

  $log->debug("Testing  if new pkg is newer and the same flavor");

  return "REBUILD" 
    if $newpkg->{'depnode'}->{'Version'}->is_newer($oldpkg->{'depnode'}->{'Version'}) 
        and $flavor eq $oldpkg->flavor();


  # nonstatic builds and no rtl packages
  $log->debug("Testing  if new pkg is non-static and no rtl's are installed");

  return "REBUILD" 
    if ! defined $static
      and $flavor ne 'noflavor'
        and ! grep { $_->pkgtype() eq 'rtl' } @$binpkgs
          and grep { $_->pkgtype() eq 'dev' } @$binpkgs;

  # static builds and no pgm_static packages
  $log->debug("Testing  if new pkg is static and no pgm_static's are installed");

  return "REBUILD" 
    if defined $static
      and $flavor ne 'noflavor'
        and ! grep { $_->pkgtype() eq 'pgm_static' } @$binpkgs
          and grep { $_->pkgtype() eq 'pgm' } @$binpkgs;

  # non static builds and no pgm packages
  $log->debug("Testing  if new pkg is non-static and no pgm's are installed");
  return "REBUILD" 
    if ! defined $static
      and $flavor ne 'noflavor'
        and ! grep { $_->pkgtype() eq 'pgm' } @$binpkgs
          and grep { $_->pkgtype() eq 'pgm_static' } @$binpkgs;


  # Play it safe for unaccounted cases
  $log->debug("Testing is indeterminate");

  return "IGNORE";
}

sub check_dependencies {
  my ($node, $flavor, $static, $installation, $log) = @_;

  $installation->cleardepenv();
  $installation->set_depenv(defined $static ? 'BuildStatic' : 'Build', 
                            $flavor);
  # Check flavored deps
  my $errors;
  my $message;

 for my $depenv('lib_link','pgm_link','compile') {
    my $deppkgs =  $installation->extract_deptree(
                                                  flavor => $flavor,
                                                  srcpkg => $node,
                                                  srcdep => $depenv,
                                                 );
      my $result = $deppkgs->check_missing(log => \$message);

    $errors++ if $result;

  }

  $log->error($message) if $errors;
  return 'FAILED' if $errors;
  return 'PASSED';

}

__END__


=head1 NAME

B<gpt-build> - Builds and installs GPT source packages and bundles. 

=head1 SYNOPSIS

B<gpt-build> [options] [macros] [source packages] flavors ...

  Options:
     -verbose                          Print copious output
     -help                             Print usage
     -man                              Print man page.
     -version                          Print GPT version.
     -installdir=path_to_installation  Override $GLOBUS_LOCATION
     -builddir=path_to_build directory Directory to unpack tarfiles
     -logdir=log_file_directory        Directory for package logs
     -srcdir=source_directory          Directory containing a source package
     -static                           Build static executables
     -force                            Force rebuilds
     -update                           Rebuild if newer
     -disable-version-checking         Ignore versions in package dependencies.
     -std-flavors                      Build all std flavors.
     -all-flavors                      Build all of the flavors
     -coresrc                          Override installed globus_core pkg
     -nosrc                            Only install the flavors
     [macro=value]                     Macro definitions
     [list of flavors to build]        Flavors that define the build options
     [list of source packages]         Tarfiles of source bundles or packages

=head1 DESCRIPTION

B<gpt-build> Builds and installs software from source
distributions.  The script reads the build instructions for each
source distribution from a package metadata file called
pkg_data_src.gpt.  If no instructions are found the script uses the
default instructions described in I<Grid::GPT::PkgMngmt::Build>.

Multiple packages can be built by passing in multiple I<-srcdir>
arguments or multiple source packages and or bundles.  For now these
have to be passed in in dependent order ie. if package foo needs
package fee's headers to build then fee should come first.

gpt-build does not create binary packages as globus-build does.  This
is done by a seperate script called I<gpt-pkg>.


=head1 FLAVORS

Build flavors are character string labels that represent a set of
compiler and linker options defined during the building of the
globus_core package.  Packages can be built with any
installed flavor.  globus_core can only be built with pre-defined
flavors.  globus_core needs to be built by hand
(i.e. ./configure|make|make install) to install a custom build flavor.


=head1 FILES IN PGM PACKAGES

Installed files such as executables that are found in pgm and
pgm_static packages will overwrite one another when multiple flavors
of the same package are built.  Thus only the files from the last
flavor built will be present after a gpt-build run. To preserve
multiple flavors of these files, they are automatically copied into
flavored subdirectories.  For example, the program
$GLOBUS_LOCATION/bin/foo will be copied into the location
$GLOBUS_LOCATION/bin/gcc32/shared if it was built with dynamically
with the gcc32 flavor.  These files are retrieved by I<gpt-pkg>.

=head1 OPTIONS

=over 8

=item B<-srcdir>

Specifies the where the source directory is.  This is used by
developers to build out of a repository..

=item B<-installdir>

Specifies the where the install directory is.  This directory will be
where the targets are installed before being archived.  Note that this
directory needs contain the installation targets of all of the
packages which satisfy build dependencies in addition to the
installation targets of the package being built.  Because of this,
installation targets are not removed after the package is archived.
Because different flavors of program packages conflict with each
other, the last flavor built is the flavor of the installed files.
For each flavor, gpt-build makes a copy of each installed file in a
program package and puts the copy in a flavored subdirectory.

=item B<-builddir>

Specifies the where the source tar files should be untarred and the
code is built.

=item B<-force>

Tells B<gpt-build> to rebuild a package even if it is already
installed.  The default behavior is to skip any packages that have
been built before.

=item B<-update>

Tells B<gpt-build> to build even if an older version is installed.

=item B<-verbose>

Prints out all of the build messages.

=item B<-std-flavors>

Build the globus_core package with all of the flavor choices marked as
standard in the flavor configuration file.  All of the other packages
will be built with all of the flavors sucessfully installed by
globus_core.

=item B<-all-flavors>

Build the globus_core package with all of the flavor choices marked as
standard in the flavor configuration file.  All of the other packages
will be built with all of the flavors sucessfully installed by
globus_core.

=item B<-nosrc>

Just install the listed flavors.  Don't build any source code.

=item B<-coresrc>

Use this version of globus_core source rather than the one found in
${datadir}/globus/gpt/globus_core-src.tar.gz

=item B<-flavor-cfg=FILE>

Use the flavor configurations found in FILE.  The default
configuration file is
${datadir}/globus/gpt/globus_flavor_labels.conf

=item B<-logdir=DIRECTORY>

Store package logs of build messages in DIRECTORY. This option also
sets -verbose. The default directory is ./logs.

=item B<-help>

Print a brief help message and exits.

=item B<-man>

Prints the manual page and exits.

=item B<-version>

Prints the version of GPT and exits.

=back

=head1 MACROS

Build macros are perl routines and variables that are used by
gpt-build while building packages.  These macros are embedded in
the build instructions part of the package's metadata file.  Some of
these macros can be set from the command line using the following
format:

  <NAME>_GPTMACRO='new value'

Note that even though this feature seems extremely flexible, it has
not been thoroughly tested mainly because the authors have not been
able to figure out what people will want to do with these macros.
Further details are found in the B<Grid::GPT::PkgMngmt::BuildMacros> manpage.


=head1 SEE ALSO

gpt-pkg(8) gpt-bundle(8) gpt-install(8) BuildMacros(3)

=head1 AUTHOR

Michael Bletzinger E<lt>mbletzin.ncsa.uiuc.eduE<gt> and Eric Blau
E<lt>blau.mcs.anl.govE<gt>

=cut