/usr/bin/colgui

#!/usr/bin/perl -w

# Copyright 2005-2010 Hewlett-Packard Development Company, L.P.
#
# colgui may be copied only under the terms of either the Artistic License
# or the GNU General Public License, which may be found in the source kit

# Debug
#    1 -  show initialization stuff
#    2 -  show ALL data input
#    4 -  show plotting parameters
#    8 -  show configuration processing
#   16 -  show configuration parameters
#   32 -  show plotting calculations (and if requested, smoothing calcs)
#   64 -  include graphic debugging on display
#  128 -  not currently used
#  256 -  do NOT start collectl.  this allows us to manually run it (or colmux)
#         on remote machine and see local debugging output.  To use this with
#         proxy, run this with at least -d257 to see colmux command.  Then on
#         proxy machine do "echo addr | colmux command" where addr is 1 addr
#  512 - don't unlink proxy file
# 1024 - see initParams.ph

use Cwd;
use Tk;
use Getopt::Long;

# The 'bundling' option makes it possible to say -sc or -i3 but then requires
# multichar names to be introduced by --.  Seems less confusing that requiring
# a space after single char options and allowing - or -- for multichar ones.
Getopt::Long::Configure ("bundling");
Getopt::Long::Configure ("no_ignore_case");

use File::Basename;
use IO::Socket;
use IO::Select;
require Tk::FileSelect;
require Tk::Dialog;

use strict;

my $LibDir='/usr/share/collectl';

my $Version="3.2.2";
my $Copyright='Copyright 2004-2010 Hewlett-Packard Development Company, L.P.';
my $License="colgui may be copied only under the terms of either the Artistic License\n";
$License.= "or the GNU General Public License, which may be found in the source kit";

# We might have been invoked via a link (at least on Linux) and we need to know
# where our physical location is.  Note we have to do this before we call initConfig()
# because we don't know where the 'require' file is located yet!
my $exeName=(!defined(readlink($0))) ? $0 : readlink($0);
my $BinDir=dirname($exeName);

# If the plot library exists in the same directory as colplot, we load it
# instead of the standard one
$LibDir=$BinDir    if -e "$BinDir/colplotlib.ph";
error("can't find $LibDir/colplotlib.ph")    if !-e "$LibDir/colplotlib.ph";
require "$LibDir/colplotlib.ph";

# set up $mycfg data structure, required by colplotlib and initparams
my $mycfg={ pcflag=>0, exename=>$exeName, bindir=>$BinDir,
	    libdir=>$LibDir, ignoreconf=>1};
initConfig($mycfg);

# When someone specifies one or more subsystems with -s, we use the following 
# to map them to plot names.
my $SubsysMap={
    c  => 'cpu',     d  => 'disk',     f  => 'nfs',     i => 'inode',  
    lc => 'lusclt',  lo => 'lusoss',   lm => 'lusmds',
    m  => 'mem',     n  => 'net',      s  => 'sock',    t => 'tcp',  
    x  => 'int',     C => 'cpudet',    D  => 'diskdet', N => 'netdet' };

# Similarly when someone specifies a plot name, we use a header prefix from 
# that entry to map it to a subsystem.  This subsystem is then used to query
# each system to see if it indeed can produce the data.
my $HeaderMap={
    CPU  => 'c',    DSK  => 'd',    INODE=> 'i',   CLT   => 'lc',
    CLTB => 'lcB',  CLTM => 'lcM',  CLTR => 'lcR', OST   => 'lo', OSTB => 'loB',
    OSTD => 'loD',  MDS  => 'lm',   MEM  => 'm',   NET   => 'n',  SOCK => 's', 
    TCP  => 't',    IB     => 'x',  ELAN => 'x',
    NFS2C =>'fc2',  NFS2S => 'fs2',  NFS3C=> 'fc3', NFS3S => 'fs3', NFS  => 'f',
    NFS2CD=>'Fc2',  NFS2SD => 'Fs2', NFS3CD=>'Fc3', NFS3SF =>'Fs3',
    'CPU:' => 'C',  'DSK:'  => 'D',   'NET:'  => 'N',    'IB:'   => 'X',
    'CLT:' => 'Lc', 'CLTB:' => 'LLc', 'CLTM:' => 'LcM',  'CLTR:' => 'LcR',
    'OST:' => 'Lo', 'OSTB:' => 'LoB', 'OSTD:' => 'LoD',  'ELAN:' => 'X' };

my ($ctrlCFlag, %hostnum, @hostname, @hostaddr, @hostnameV);
my (%addrhost, %sockHandle);
my (@c, @xLast, @yLast, @vLast, @curPoint, @maxPoints);
my ($Record, $frame, $frame2, $select, $socket);

# These are specifically needed to retain global values for plotting
my ($xLast, $yLast, $lastHigh);

select;
$|=1;

my $Route=   '/sbin/route';
my $Ifconfig='/sbin/ifconfig';
my $myHost=`hostname`;
chomp $myHost;
$myHost=~s/\..*//;    # get rid of domain if there

my ($plotYMax, $plotCols, $plotNames, $dataDiv);
$plotYMax=$plotCols=$plotNames=$dataDiv=0;

my (@plotColor, $plotBorder);
$plotColor[0]='blue';
$plotColor[1]='black';
$plotColor[2]='red';
$plotColor[3]='orange';
$plotColor[4]='white';
$plotColor[5]='yellow';
$plotColor[6]='magenta';
$plotColor[7]='green';
$plotBorder='black';

my ($labelFont, $timeFont);
$labelFont='Courier -12 bold';
$labelFont='Helvetica -12 bold';
$timeFont='Helvetica -10 bold';

# Trying to make the display geometry work for all combinations of parameters
# is hopeless (or at least not worth all the effort).  It seems to have to do with
# the number of pixels in lines, borders, etc and the round-off when odd or even.
# Things currently look good with the following numbers so I'm leaving them that way.
my $bw=5;         # border width
my $ti=1;         # text indent
my $th=13;        # text height
my $tMajL=3;      # major timer tick length
my $tMajW=3;      # major timer tick width
my $tMajC=1;      # major timer tick color
my $tMinL=2;      # minor timer tick length
my $tMinW=2;      # minor timer tick width
my $tMinC=0;      # minor timer tick color
my $tw=9;         # text width (approx);
my $bg='linen';   # background
my $tfw=$bw*2+10; # left-hand title frame width (not including borders)
my $rectWidth=2;

my ($swXPad, $swYPad, $neXPad, $neYPad);
$swXPad=40;
$neXPad=10;

my $IgnoreData='waiting for|^\s*$';
my $SubsysValid='cdfilmnsxCDLN';

#    I n i t i a l i z a t i o n s

my $debug=0;
my $tmpFile=  "/tmp/colgui-$$.tmp";
my $proxyFile="/tmp/colgui-$$.proxy";

# Declares for options
my ($address, $colbin, $colopt_O, $count, $geometry);
my ($hFlag, $xFlag, $hosts, $interval, $lineWidth, $logdir);
my ($lustype, $lusopt, $machines, $nfstype, $notimeFlag, $numPerRow);
my ($plotType, $plotWidth, $port, $proxy, $colmuxbin, $homoFlag);
my ($radInterval, $realAddress, $rsh, $filter, $showParamsFlag, $showPlotsFlag);
my ($showaddrs, $smooth, $timeFormat, $timeZeroFlag, $username);
my ($xAxis, $yAxis, $vFlag);

$lineWidth=1;
$plotWidth=3;

$colbin='collectl';
$geometry='n';
$interval=1;
$lustype=$lusopt='';
$numPerRow=4;
$port=2655;
$colmuxbin='/usr/sbin/colmux';
$smooth=0;
$timeFormat='20:4';
$xAxis=60;
$yAxis=50;

my ($plots, $subsys)=('','');
GetOptions("d=i"  => \$debug,
	   "i=f"  => \$interval,
	   "r=i"  => \$numPerRow,
	   "p=s"  => \$plots,
	   "s=s"  => \$subsys,
	   "h!"   => \$hFlag,
           "x!"   => \$xFlag,
	   "v!"   => \$vFlag,
	   "address=s"    => \$address,
	   "colbin=s"     => \$colbin,
	   "colopt_O=i"   => \$colopt_O,
	   "colmuxbin=s"  => \$colmuxbin,
	   "count=i"      => \$count,
	   "filter=s"     => \$filter,
	   "nfstype=s"    => \$nfstype,
	   "geometry=s"   => \$geometry,
	   "help!"        => \$hFlag,
	   "hosts=s"      => \$hosts,
	   "linewidth=i"  => \$lineWidth,
	   "logdir=s"     => \$logdir,
	   "lustype=s"    => \$lustype,
	   "machines=s"   => \$machines,
           "homogeneous!" => \$homoFlag,
	   "notime!"      => \$notimeFlag,
	   "plottype=s"   => \$plotType,
	   "plotwidth=i"  => \$plotWidth,
	   "port=s"       => \$port,
	   "proxy=s"      => \$proxy,
	   "radint=i"     => \$radInterval,
	   "realaddress=s"=> \$realAddress,
	   "rsh=s"        => \$rsh,
	   "showaddrs!"   => \$showaddrs,
	   "showparams!"  => \$showParamsFlag,
	   "showplots!"   => \$showPlotsFlag,
	   "smooth=f"     => \$smooth,
	   "timeformat=s" => \$timeFormat,
	   "timezero!"    => \$timeZeroFlag,
	   "username=s"   => \$username,
	   "xaxis=i"      => \$xAxis,
	   "yaxis=i"      => \$yAxis,
	   ) or error("type -h for help");
$mycfg->{debug}=$debug;

help()                if defined($hFlag);
helpAdvanced()        if defined($xFlag);
showplots($mycfg)     if defined($showPlotsFlag);

error("V$Version\n\n$Copyright\n$License")   if defined($vFlag);

error("you cannot specify more than one of -address, -hosts or -machines")
    if defined($address)+defined($hosts)+defined($machines)>1;
error("you must specify exactly one of -address, -hosts or -machines with -proxy")
    if defined($proxy) && defined($address)+defined($hosts)+defined($machines)!=1;
error("can't file file '$machines'")
    if defined($machines) && !-e $machines;
error("valid -nfstype is 'c', '3', or 'c3'")
    if defined($nfstype) && $nfstype!~/^[c3]+$/i;

# need more than just -sf to do nfs.  Also need --nfsfilt and this is easier!
error("nfs plotting currently not supported via -s.  use -p")    if $subsys=~/f/i;

# Default parameters
if ($subsys eq '' && $plots eq '')
{
  $subsys='cdn';
  $numPerRow=3;
  $timeZeroFlag=1;
}

my $timeFlag=!$notimeFlag;
error("invalid value of '$subsys' for -s must be subset of '$SubsysValid'")
    if $subsys ne '' && $subsys!~/^[$SubsysValid]+$/;

error("valid values for -g are 'n', 'c, 'nd' and 'cd'")
    if $geometry ne 'n'  && $geometry ne 'c' && 
       $geometry ne 'nd' && $geometry ne 'cd';

# Valid plot types, but only if specified
if (defined($plotType))
{
  error("plot type must contain l, p or b")  if $plotType!~/[lpb]{1}/i;
  error("bar plots are already stacked")     if $plotType=~/b/i && $plotType=~/s/i;
  error("incorrect plot type")               if $plotType!~/^[lpbsr]+$/;
  error("no radial bar plots!")              if $plotType=~/r.*b|b.*s/;
}

# Verify list of plots and expand any macros
my $paramsConfig={};
exit(1)    if !initParamsInit($mycfg, $paramsConfig);

# We're not even going to worry about dups yet as we do that later
my ($tempPlots, $nfsFlag, $nfsdFlag)=('',0,0);
foreach my $plot (split(/,/, $plots))
{
  # NOTE - nfs plot names MUST contain 'nfsV' and detail plots a 'd'
  # I really hate to do this but it's easy as long as I control the 'defs'.
  $nfsFlag++     if $plot=~/nfsV/ && $plot!~/d/;
  $nfsdFlag++    if $plot=~/nfsV/ && $plot=~/d/;
  error("can only have 1 nfs summary or detail plot")
      if $nfsFlag>1 || $nfsdFlag>1;

  if (defined($paramsConfig->{AllPlots}->{$plot}))
  {
    $tempPlots.="$plot,";
    next;
  }

  error("invalid plot '$plot'")    if !defined($paramsConfig->{Macros}->{$plot});
  foreach my $name (split(/,/, $paramsConfig->{Macros}->{$plot}))
  {
    $tempPlots.="$name,";
  }
}
$tempPlots=~s/,$//;
$plots=$tempPlots;

#    B u i l d    L i s t    O f    M a c h i n e s

my ($ssh, $numHosts, $command, $line, $netaddr);
$ssh=(defined($rsh)) ? 'rsh' : 'ssh';
$username=(defined($username)) ? "$username\@" : '';
if (defined($machines))
{
  readMachines($machines);
}
elsif (defined($hosts))
{
  readMachines('/etc/hosts', $hosts);
}
else
{
  my ($addr, $hostV, $hostNum);
  $hostNum=0;
  $address=$myHost    if !defined($address);
  $address=~s/\s+/,/g;
  foreach $addr (split(/,/, $address))
  {
    ($addr, $hostV)=split(/:/, $addr);
    $hostV=$addr    if !defined($hostV);

    my $hostent=gethostbyname($addr);
    error("can't find address for $addr")    if !defined($hostent);
    $netaddr=inet_ntoa(scalar($hostent));
    push @hostaddr, $netaddr;
    push @hostname, $addr;
    $addrhost{$netaddr}=$hostNum++;

    # if address of form addr1:addr2, use addr2 for the vertical form
    # of the name with --geo nd
    $hostV=~s/(.{1})/$1\n/g;
    $hostV=~s/\n$//;
    push @hostnameV, $hostV;
  }
}
$numHosts=scalar(@hostname);

my $i;
if ($showaddrs)
{
  for ($i=0; $i<$numHosts; $i++)
  {
    print "$hostaddr[$i]\n";
  }
  exit;
}

#    Q u e r y    H o s t s    F o r    C o n f i g

my $errors;
my $maxvals={};
my ($colver, $memory);

# Since success is 0, we need to use '&&'
my $globals={};
my $hostsConfig={};
getRemoteConfig($hostsConfig, $globals) &&
   error("Cannot continue unless all hosts reachable and properly configured");

#    G e t  /  V e r i f y    P l o t t i n g    I n f o

# Build up a list of all the requested plots
my (@plots, %plotsSeen);
foreach (my $i=0; $i<length($subsys); $i++)
{
  my $tempsys=substr($subsys, $i, 1);
  if ($tempsys!~/l/i)
  {
    push @plots, $SubsysMap->{$tempsys};
  }
  else  # Handle lustre plots on a per server type basis
  {
    for (my $j=0; $j<length($globals->{alltypes}); $j++)
    {
      my $lustype=substr($globals->{alltypes}, $j, 1);
      push @plots, $SubsysMap->{"$tempsys$lustype"};
    }
  }
  $plotsSeen{$plots[$i]}=1;
}

foreach my $plot (split(/,/, $plots))
{
  # Silently ignore dups...
  next    if defined($plotsSeen{$plot});

  error("invalid plot '$plot'")    if !defined($paramsConfig->{AllPlots}->{$plot});
  push @plots, $plot;
  $plotsSeen{$plot}=1;
}

# Finally add to the list of selected subsystems (if not already there) all the
# possible subsystems associated with each plot because we need to use this to
# query the target systems with later when we ask for their headers AND reset
# $plots to contain the complete list.
$plots='';
my $nfsFilt='';
my $lustOpts='';
my $alltypes=$globals->{alltypes};
foreach my $plot (@plots)
{ 
  # All plots should contain valid headers, but custom ones may not...
  $paramsConfig->{AllPlots}->{$plot}->{yname}=~/\[(.*?)\]/;
  my $tempsys=$HeaderMap->{$1};
  error("Can't find plot definition for column header '[$1]'")
      if !defined($tempsys);

  # Special processing for nfs since we need to set --nfsopts based on plotname
  # format of tempsys is "sxxx" where 's' is subsys and 'xxx' are its options
  if ($tempsys=~/f/i)
  {
    $nfsFilt=substr($tempsys, 1);
    $tempsys=substr($tempsys, 0, 1);
  }

  # Special processing for lustre since the subsys also contains the type
  my ($temptype, $tempopt);
  if ($tempsys=~/l/i)
  {
    # ...but if none of the systems are running lustre don't bother
    next    if $alltypes!~/[cmo]/i;

    # By design, both MDS and OSS use the same detail file for -OD 
    # (since they could be on the same server) and that's the one
    # associated with OSS.  So, if we see D and don't have a server type 
    # of 'o' selected, we need to force the type to 'm' so we can match.
    # Only then, if an mds wasn't selected, will the plot will be ignored.
    $tempopt =substr($tempsys,2,1);
    $temptype=substr($tempsys,1,1);
    $temptype='m'    if $temptype eq 'o' && $alltypes!~/o/ && $tempopt eq 'D';

    $tempsys= substr($tempsys,0,1);
    $lustOpts=$tempopt     if $lustOpts!~/$tempopt/;

    if ($alltypes!~/$temptype/)
    {
      print "Silently ignoring plots for type '$temptype'\n"    if $debug & 1;
      next;
    }
  }
  $subsys.=$tempsys    if $subsys!~/$tempsys/;
  $plots.="$plot,";
}
$plots=~s/,$//;

# We now have a list of all the plots we want to draw as well as the 
# invidual system configurations so we can now grab the headers for each
# system based on them.
getRemoteHeader($hostsConfig, $subsys, $lusopt, $lustOpts, $globals) &&
   error("Cannot continue unless all hosts reachable and properly configured");

# set up plotting parameters based on $subsys, $plots and $machines
my (%plots, $results, $status);
my ($rectFlag, $radialFlag, $numPlots);
my ($yMajWidth, $yMinWidth, $maxPlots)=(0,0,0);

#    G e t    P l o t t i n g    P a r a m e t e r s

my (%plotinfo, %globinfo);
foreach my $addr (@hostaddr)
{
  $plotinfo{$addr}=[];
  $globinfo{$addr}={};
  my $header=$hostsConfig->{$addr}->{header};
  my $select={filename=>'', plots=>$plots, filters=>$filter, plottype=>$plotType};
  $numPlots=initParams($mycfg, $paramsConfig, $select, \$header, $plotinfo{$addr}, $globinfo{$addr});
  error("no plots selected for $addr")           if !$numPlots;
  showParams($mycfg, $plotinfo{$addr}, $addr)    if $showParamsFlag || $debug & 1;

  # Need to know some plot/system-wide parameters
  $yMajWidth= $globinfo{$addr}->{ymajwidth}
                         if $yMajWidth<$globinfo{$addr}->{ymajwidth};
  $yMinWidth= $globinfo{$addr}->{yminwidth}
                         if $yMinWidth<$globinfo{$addr}->{yminwidth};
  $maxPlots=$numPlots    if $numPlots>$maxPlots;
  $radialFlag=1          if $globinfo{$addr}->{radialflag};
  $rectFlag=1            if $globinfo{$addr}->{rectflag};
}
exit    if $showParamsFlag;

# relatively rare situation -- If doing "--geo cd" detail plots for 
# multiple systems AND first system has filler entries, the headers over
# those will be bogus, so use the values for the first legit system we
# encounter.  It ain't perfect, but it is about the best I can think of
goto skip;
for (my $i=0; $i<scalar(@{$plotinfo{$hostaddr[0]}}); $i++)
{
  # look for any undefined plot names
  my $plotref=$plotinfo{$hostaddr[0]}->[$i];
  if (!defined($plotref->{PlotName}))
  {
    # now look through the rest of the hosts for the first one that
    # has a legit entry
    for (my $j=1; $j<scalar(@hostaddr); $j++)
    {
      my $tmpref=$plotinfo{$hostaddr[$j]}->[$i];
      if (defined($tmpref->{Name}))
      {
	$plotref->{Name}=     $tmpref->{Name};
	$plotref->{YMinLabel}=$tmpref->{YMinLabel};
	$plotref->{YMaxLabel}=$tmpref->{YMaxLabel};
	last;
      }
    }
  }
}

skip:
# Just can't find a less dense way to display multiple plots with -gcd`
error("-gcd doesn't apply to multiple plots")
    if $geometry eq 'cd' && $numPlots>1;

# Very special but necessary test.
error("Number of plots ['$numPlots'] exceeds the number of plots per row ".
      "['$numPerRow'] for --geo nd.\nEither remove 'd' from -geo or increase '-r'")
          if $geometry eq 'nd' && $numPlots > $numPerRow;

my $firstPassFlag=1;  # cleared when a full xaxis drawn
my ($major, $minor, $tickMajFreq, $tickMinFreq);
if ($timeFlag)
{
  ($major, $minor)=split(/:/, $timeFormat);
  $minor=(defined($minor)) ? $minor : $major;
  error("-x, which is $xAxis, must be a multiple of --timeformat which is '$timeFormat'")
      if int($xAxis/$major)*$major!=$xAxis;
  error("major tick marks must be multiple of minor ones")
      if int($major/$minor)*$minor!=$major;
  $tickMajFreq=$major;
  $tickMinFreq=$minor;
}

# the radial plots are based on the yaxis height
my $radius=$yAxis/2;
my $plotDegr=(!defined($radInterval)) ? 360/$xAxis : 360/$radInterval;

# determine the actual size of the xaxis based on the width of a plot point,
# that is the horizontal distance between 2 points
$xAxis=$xAxis*$plotWidth;

# This is tricky as neYPad depends on several things and other things depend on it!
# so, we'll handle ALL cases here and now.
$neYPad=0;
$neYPad=$th+2        if $geometry ne 'nd';  # 1 row of text

my ($xCenter, $yCenter);
$neXPad+=$yMinWidth*$tw;
$xCenter= $bw+$swXPad+$radius;
$xCenter-=$swXPad               if $geometry=~/d/;
$yCenter= $bw+$neYPad+$radius;
#$yCenter-=$bw+$th-$rectWidth    if $geometry=~/d/ && $geometry!~/c/;

#    O p e n    S o c k e t    F o r    L i s t e n i n g

my $remoteStarted=0;
my ($host, $cmdPipe);

$socket = new IO::Socket::INET(Type=>SOCK_STREAM,
                               Reuse=>1, Listen => 1,
                               LocalPort => $port)
      or error("colgui: couldn't create local socket on port: $port");
$select = new IO::Select($socket);
print "colgui: Listening for connections on port: $port\n"
    if $debug & 1;

#    S t a r t    D a t a    C o l l e c t i o n

# collectl switches
my $switches;
my $subOpts='';

# Now start up collectl one machine at a time or else have the proxy do it!
my $error='';
if (!defined($proxy))
{
  foreach my $addr (@hostaddr)
  {
    # Use the type for this specific address.
    my $lustype=$hostsConfig->{$addr}->{lustype};

    # set lustre options based on type of this system
    my $thisLustOpts='';
    $thisLustOpts='MR'  if $lustype eq 'c';
    $thisLustOpts='B'   if $lustype eq 'o';
    $thisLustOpts=''    if $lustype eq 'm';

    # we need a space after -s in case something like 'l'.
    $switches= "-i$interval -s $subsys -P";
    $switches.=" -c$count"                 if defined($count);
    $switches.=" --lustsvc $lustype"       if $subsys=~/l/;
    $switches.=" --lustopts $thisLustOpts" if $thisLustOpts ne '';
    $switches.=" --nfsfile $nfsFilt"       if $nfsFilt ne '';
    $switches.=" -f $logdir -F0"           if defined($logdir);

    # WARNING - at one point I redirected stdout to /dev/null too and on some
    # systems this caused the select loop to never wake up!
    my $myReturnAddr=getReturnAddress($addr);
    $command=(defined($rsh)) ? 'rsh ' : 'ssh ';
    $command.="$username$addr $colbin $switches -A $myReturnAddr:$port";
    $command.=" 2>/dev/null"       if !$debug;
    print "Command: $command\n"    if $debug & 1;
    system("$command &");
    $remoteStarted++;
  }
}
else
{
  # First, save a copy of the hosts selected, for passing to colmux.
  my $firstAddr;
  open TMP, ">$proxyFile" or error("Couldn't create temp file '$proxyFile'");
  foreach my $addr (@hostaddr)
  {
    $firstAddr=$addr    if !defined($firstAddr);
    print TMP "$addr\n";;
  }
  close TMP;

  # Next, we need to add the  target host setting for the proxy itself, so that
  # when it connects back, we'll recognize it.  We don't care about the host
  # number since it will never show up in the return record where the actual
  # remapping takes place.
  my $myReturnAddr=getReturnAddress($proxy);
  $addrhost{$proxy}=999;
  $addrhost{$myReturnAddr}=0;

  $command="cat $proxyFile | $ssh $username$proxy ";
  $command.="\"$colmuxbin -proxy $myReturnAddr -port $port ";
  $command.="-collectl $colbin "         if defined($colbin);
  $command.="-command '$switches'\"";
  $command.=" 1>$tmpFile 2>&1";
  print "Command: $command\n"            if $debug & 1;

  # we need to make sure command succeeded.  1 second seems to be enough
  # time to let error file get created.  It also gives enough time to the
  # proxy file to be deletable.  We also want to to this way so we only
  # delete file in one place
  if (!($debug & 256))
  {
    system("$command &");
    sleep 1;
    $error=`cat $tmpFile`;
    print $error    if $error!~/waiting|Warning/;    # normal startup messages
  }
}
unlink $tmpFile;
unlink $proxyFile    unless $debug & 512;

 #    B u i l d    L a y o u t

# A few overrides for the 'dense' displays
if ($geometry=~/d/)
{
  # Remember, neYPad still needs room for title
  $swXPad=$neXPad=0;

  # Not totally obvious, but in dense mode 'numPerRow' controls the 
  # width of the header as well as limiting the number of time axises 
  # to the first row so reset it to reality.
  $numPerRow=$numHosts*$numPlots
      if $geometry eq 'cd' && $numHosts*$numPlots<$numPerRow;
  $numPerRow=$numPlots
      if $geometry eq 'nd' && $numPlots<$numPerRow;
}
printf "NumPlots: %d NumHosts: %d NumPerRow: %d xAxis: %d yAxis: %d\n",
    $numPlots, $numHosts, $numPerRow, $xAxis, $yAxis    if $debug & 1;

my $top = MainWindow->new;

# Turns out with the xAxis size now being time based, radial plots break
# because they share code with xy plots AND use xAxis.  To make a long
# stort the following is a hack that should be cleaned up some time.
my $xAxisSave=$xAxis;

# Top row is special for 'dense' geometries.
my ($xyInst, $radialInst);
displayInit(-1,0)    if $geometry=~/d/;

for ($host=0; $host<$numHosts; $host++)
{
  my $addr=$hostaddr[$host];
  my $plotref=$plotinfo{$addr};

  for (my $plot=0; $plot<$numPlots; $plot++)
  {
    # Note - plottypes are initially assigned by plot number and we
    # need to propogate to all instances
    my $inst=$host*$numPlots+$plot;
    last    if !defined(@$plotref[$plot]);

    $maxPoints[$inst]=(@$plotref[$plot]->{PType}!~/r/) ?
	int($xAxis/$plotWidth) : int(360/$plotDegr);
    printf "Init -- %d %d Addr: %s Plottype: %s Num-YAxis: %d MaxPoints: %d\n",
            $host, $plot, $addr,
            @$plotref[$plot]->{PType}, scalar(@{@$plotref[$plot]->{YMax}}),
            $maxPoints[$inst]  
		if $debug & 1;

    # Init Plotting grid and set $curPoint to it's max so that when we
    # go to plot the first element it will get initialized properly.
    displayInit($host, $plot);
    $curPoint[$inst]=$maxPoints[$inst];
  }
}

# Add a title, including time(s) to fill a graph
my $titleData="Refresh: $interval sec";
$titleData.=sprintf("  X-Axis Sweep: %s", toTime($maxPoints[$xyInst]*$interval))
		    if $rectFlag;
$titleData.=sprintf("  Radial Sweep: %s", toTime($maxPoints[$radialInst]*$interval))
		    if $radialFlag;
$top->title("colgui - $titleData");

# This makes it all work by telling TK to start the 'getData' processing
my $timeStart=time;
$frame->after(1000, \&getData);
$ctrlCFlag=0;
$SIG{"INT"}= \&sigInt;
$SIG{"PIPE"}=\&sigPipe;

foreach my $addr (@hostaddr)
{
  # Report in text form what the user is seeing for those cases where all
  # the labels aren't visible!
  print "*** Plotting Attributes for $addr ***\n";
  printf "%-23s", "Colors:";
  for (my $i=0; $i<scalar(@plotColor); $i++)
  {
    printf " %-8s", $plotColor[$i];
  }
  print "\n";

  my $plotref=$plotinfo{$addr};
  for (my $i=0; $i<$numPlots; $i++)
  {
    # note that if filler we've already reset the values to something legal
    # AND that not all plots are defined for all systems
    last    if !defined(@$plotref[$i]);
    printf "Labels For %-12s", "@$plotref[$i]->{Name}:";

    # Only report those columns that begin what might be a compound expression
    my $colNum=0;
    my $cLabelIndex=0;
    my $lastOper=' ';
    my $numCols=scalar(@{@$plotref[$i]->{YColnum}});
    my @cLabels=split(/,/, @$plotref[$i]->{CLabelStr});
    for (my $j=0; $j<$numCols; $j++)
    {
      my $cLabel;
      my $oper=@$plotref[$i]->{Oper}->[$j];

      if ($oper ne ' ')
      {
	$lastOper=$oper;
	next;
      }

      # If coming out of a compound expression we need to use one of the clabels 
      # instead of 'standard' ones.  But in either case, they both count as a column
      printf " %-8s", ($lastOper eq ' ') ? 
	  @$plotref[$i]->{YLabels}->[$colNum] : $cLabels[$cLabelIndex++];
      $colNum++;
    }
    print "\n";
    last    if $homoFlag;
  }
}

# Fire things off in an event driven spin loop
my $numConnections=0;
Tk::MainLoop;

sub displayInit
{
  my $hostNum=shift;
  my $plotNum=shift;
  my ($inst, $temp, $rowNum);
  my ($x, $y, $title);

  # NOTE - main header based on first host.  All settings should be
  # the same...
  my $addr=($hostNum!=-1) ? $hostaddr[$hostNum] : $hostaddr[0];
  my $plotref=$plotinfo{$addr};

  if ($hostNum==-1)
  {
    $rowNum=0;
  }
  else
  {
    $inst=$hostNum*$numPlots+$plotNum;
    $rowNum=int($inst/$numPerRow);
  }
  print "displayInit - Host: $hostNum Addr: $addr  Plot: $plotNum\n"
      if $debug & 1;

  # Start out with enough for legend, but if doing time, we need more and for
  # dense geometries we only include time if first row of display.
  $swYPad=$th+$ti;
  $swYPad+=2    if $timeFlag;
  $swYPad=0     if $geometry=~/d/ && (!$timeFlag || $rowNum>0);

  # The big hack to make a local copy!!!
  my $xAxis=(@$plotref[$plotNum]->{PType}!~/r/) ? $xAxisSave : $yAxis;

  # These are the dimensions for each graphic window, excluding borders
  my $width= $xAxis+$swXPad+$neXPad+$rectWidth*2;
  my $height=$yAxis+$swYPad+$neYPad+$rectWidth*2+$lineWidth;
  #print "RECTD: $rectWidth, NEXPAD: $neXPad, NEYPAD: $neYPad, YAXIS: $yAxis  XAXIS: $xAxis\n";
  #print "INST: $inst  ROW: $rowNum  SWXPAD: $swXPad  SWYPAD: $swYPad  WIDTH: $width HEIGHT: $height\n";

  ###############################################
  #    B u i l d    T o p    H e a d e r    R o w
  ###############################################

  # ...but only on request
  if ($hostNum==-1)
  {
    if ($geometry eq 'nd')
    {
      $frame=$top->Frame();
      $frame->pack(-side=>'top', -anchor=>'n');
      for (my $plot=-1; $plot<$numPlots; $plot++)
      {
        my $temp=$frame->Canvas(-width=>($plot==-1)? $tfw:$width, 
			     -height=>40, -relief=>'groove',
			     -bd=>$bw, -bg=>$bg);
	if ($plot!=-1)
        {
	  my $plotName=@$plotref[$plot]->{Name};
	  my $plotInst=@$plotref[$plot]->{InstName};
	  my $plotType=@$plotref[$plot]->{PType};
	  my $plotYMin=@$plotref[$plot]->{YMin};
	  my $plotYMax=@$plotref[$plot]->{YMax};

	  my $numcols=  scalar(@{@$plotref[$plot]->{YColnum}});
	  my $numYLabel=scalar(@{@$plotref[$plot]->{YMinLabel}});

	  # Plot Name(s)
	  $x=$y=$bw+$ti;
  	  $temp->createText($x, $y, -font=>$labelFont, -anchor=>'nw',
			    -text=>@$plotref[$plot]->{Name},
			    -justify=>'left');

	  # Y-Axis Labels as a string
	  my $yRange='';
	  for (my $i=0; $i<$numYLabel; $i++)
	  {
	    $yRange.=sprintf("[%d,%d] ", 
		       @$plotref[$plot]->{YMinLabel}->[$i],
		       @$plotref[$plot]->{YMaxLabel}->[$i]);
	  }
	  $y+=$th;
  	  $temp->createText($x, $y, -font=>$labelFont, -anchor=>'nw',
			    -text=>"YAxis: $yRange");

	  # Line Names
	  $y+=$th;
	  for (my $col=0; $col<$numcols; $col++)
          {
	    my $colName=@$plotref[$plot]->{YLabels}->[$col];
	    $temp->createText($x, $y, -font=>$labelFont, -anchor=>'nw',
			      -fill=>$plotColor[$col], -justify=>'left',
			      -text=>$colName);
	    $x+=(length($colName)+1)*$tw;
          }
        }
	$temp->pack(-side=>'left');
      }
    }
    elsif ($geometry eq 'cd')
    {
      my $plotName=@$plotref[$plotNum]->{Name};
      my $plotInst=@$plotref[$plotNum]->{InstName};
      my $plotType=@$plotref[$plotNum]->{PType};
      my $plotYMin=@$plotref[$plotNum]->{YMin};
      my $plotYMax=@$plotref[$plotNum]->{YMax};
      
      my $numcols=  scalar(@{@$plotref[$plotNum]->{YColnum}});
      my $numYLabel=scalar(@{@$plotref[$plotNum]->{YMinLabel}});

      # One common 'width' for headers as well as graphs, but remember it 
      # doesn't include border width. But also remember bw is included
      # with canvas already 
      $x=($width+$bw*2)*$numPerRow-$bw;

      # Y-Axis Labels as a string
      my $yRange='';
      for (my $i=0; $i<$numYLabel; $i++)
      {
	$yRange.=sprintf("[%d,%d] ", 
		       @$plotref[$plotNum]->{YMinLabel}->[$i],
		       @$plotref[$plotNum]->{YMaxLabel}->[$i]);
      }
      $title="Plot: @$plotref[0]->{Name} Y-Range: $yRange";

      $frame=$top->Frame();
      $frame->pack(-side=>'top', -anchor=>'n');
      $temp=$frame->Canvas(-width=>$x, -height=>$bw+$th, 
			   -relief=>'groove', -bd=>$bw, -bg=>$bg);
      $temp->createText($bw+$ti, $bw+$ti, -font=>$labelFont,
			-text=>$title, -anchor=>'nw', -justify=>'left');

      # Legend goes right after 'Range' and the 'x' calculations are
      # pretty lame if I do say so myself!
      $x=length($title)*$tw*.8 ;
      $temp->createText($x, $bw+$ti, -font=>$labelFont, -text=>'Legend:',  
			-anchor=>'nw', -justify=>'left');

      $x+=length('Legend')*$tw;  # leave off ': ' to acct for narrow chars

      my $col=0;
      my $lastOper=' ';
      my $cLabelIndex=0;
      my @cLabels=split(/,/, @$plotref[$plotNum]->{CLabelStr});
      for (my $i=0; $i<$numcols; $i++)
      {
        my $oper=@$plotref[$plotNum]->{Oper}->[$i];
        if ($oper ne ' ')
	{
	  $lastOper=$oper;
	  next;
	}

	my $colName=($lastOper eq ' ') ?
	    @$plotref[$plotNum]->{YLabels}->[$col] : $cLabels[$cLabelIndex++];
        $temp->createText($x, $bw+$ti, -font=>$labelFont,
			  -fill=>$plotColor[$col], 
			  -anchor=>'nw', -justify=>'left',
			  -text=>$colName); 
	$x+=(length($colName)+1)*$tw;
	$col++;
      }
      $temp->pack();
    }
    return;
  }

  ############################################
  #    B u i l d    D i s p l a y    B o d y
  ############################################

  my $plotName=@$plotref[$plotNum]->{Name};
  my $plotInst=@$plotref[$plotNum]->{InstName};
  my $plotType=@$plotref[$plotNum]->{PType};
  my $plotYMin=@$plotref[$plotNum]->{YMin};
  my $plotYMax=@$plotref[$plotNum]->{YMax};

  my $numcols=  scalar(@{@$plotref[$plotNum]->{YColnum}});
  my $numYLabel=scalar(@{@$plotref[$plotNum]->{YMinLabel}});

  # This code always starts a new row!  The mechanism for deciding when
  # to start a new row is:
  #   - In normal mode, every time the host changes (plotNum==0) 
  #     or we exceed row limit
  #   - In compact mode, when current the one fills up.
  if (($geometry=~/n/ && ($plotNum % $numPerRow)==0) ||
      ($geometry!~/n/ && ($inst % $numPerRow)==0))
  {
    print "Create Top Level Frame\n"    if $debug & 1;
    # I hate TK!!!  We need a frame for the whole row, such that each
    # is packed on top of each other.  Then we need a second frame inside of
    # that which is packed to the left.  Maybe there's an easier way but
    # I don't know it!
    $frame=$top->Frame();
    $frame->pack(-side=>'top', -anchor=>'n');
    $frame2=$frame->Frame();

    # First element for normal-dense geometry of first row is special;
    if ($geometry eq 'nd')
    {
      # for this format we use the 'vertical' form of the hostname which
      # comes from the 2nd column of the machines file (or the 3rd if it 
      # exists)
      $temp=$frame->Canvas(-width=>$tfw, -height=>$height,
			   -relief=>'groove', -bd=>$bw, -bg=>$bg);
      $temp->createText($bw+$tfw/2, $neYPad+$yAxis+$swYPad+$bw,
			-font=>$labelFont, -anchor=>'s', 
			-text=>$hostnameV[$hostNum]);
      $temp->pack(-side=>'left');
    }
  }

  #    P l o t    P r o p e r

  $c[$inst] = $frame2->Canvas(-width=>$width, -height=>$height, 
			      -relief=>'groove', -bd=>$bw, -bg=>$bg); 
  if ($plotName eq '')
  {
    $c[$inst]->pack(-side=>'left');
    $frame2->pack(-side=>'left');
    return;
  }

  ########################
  # RECTANGLE OR CIRCLE
  ########################

  if ($plotType!~/r/)
  {
    $xyInst=$inst;
    my $rw=$rectWidth;
    $c[$inst]->createRectangle($bw+$swXPad+1,            $bw+$neYPad+1,
			       $bw+$swXPad+$xAxis+$rw*2, $bw+$neYPad+$yAxis+$rw+1,
			       -outline=>$plotBorder, -width=>$rectWidth);
  }
  else
  {
    $radialInst=$inst;
    $c[$inst]->createOval($swXPad+$bw+1, $neYPad+$bw+1,
                          $swXPad+$radius*2+$bw, $neYPad+$radius*2+$bw,
                          -outline=>$plotBorder, -width=>$rectWidth);

    $c[$inst]->createOval($xCenter-1, $yCenter-1, $xCenter+1, $yCenter+1,
			  -outline=>'blue', -fill=>'blue', -width=>$rectWidth);

  }

  #######################
  # TITLE AND AXIS
  #######################

  if ($geometry eq 'n' || $geometry eq 'c')
  {
    # If time axis, put title and legend in title bar
    my $col=0;
    my $lastOper=' ';
    my $cLabelIndex=0;
    my @cLabels=split(/,/, @$plotref[$plotNum]->{CLabelStr});
    $title=$hostname[$hostNum];
    $title.=": $plotName";
    $title.=":$plotInst"    if $plotInst ne '';
    if ($timeFlag && $plotType!~/r/)
    {
      $x=$bw+1+$swXPad;
      $c[$inst]->createText($x, $bw+$ti*2, 
			    -font=>$labelFont, -anchor=>'nw', -justify=>'left',
			    -text=>$title, -tag=>"title-$inst");

      # Start next field 1 char width to the right of the title.
      my $xpos=($c[$inst]->bbox("title-$inst"))[2];
      $x=$xpos+$tw;
      $y=$neYPad+$yAxis+2;
      for (my $i=0; $i<$numcols; $i++)
      {
        my $oper=@$plotref[$plotNum]->{Oper}->[$i];
        if ($oper ne ' ')
        {
          $lastOper=$oper;
          next;
        }

        my $colname=($lastOper eq ' ') ?
            @$plotref[$plotNum]->{YLabels}->[$col] : $cLabels[$cLabelIndex++];
        $c[$inst]->createText($x, $bw+$ti*2,
			    -font=>$labelFont, 
			    -anchor=>'nw', -fill=>$plotColor[$col],
			    -text=>$colname);
        $x+=(length($colname)+1)*$tw;
        $col++;
      }
    }
    # separate title and legend
    else
    {
      $c[$inst]->createText($bw+1+$swXPad+$xAxis/2, $bw+$ti*2, 
			  -font=>$labelFont, 
			  -anchor=>'n', -text=>$title);

      $x=$swXPad;
      $y=$neYPad+$yAxis+$rectWidth*2+2;
      for (my $i=0; $i<$numcols; $i++)
      {
        my $oper=@$plotref[$plotNum]->{Oper}->[$i];
        if ($oper ne ' ')
        {
          $lastOper=$oper;
          next;
        }

        my $colname=($lastOper eq ' ') ?
            @$plotref[$plotNum]->{YLabels}->[$col] : $cLabels[$cLabelIndex++];
        $c[$inst]->createText($x+$bw+1, $y+$bw,
			    -font=>$labelFont, 
			    -anchor=>'nw', -fill=>$plotColor[$col],
			    -text=>$colname);
        $x+=(length($colname)+1)*$tw;
        $col++;
      }
    }

    # *** MAIN Y-AXIS ***
    my $yIncr=($plotType=~/r/) ? $radius : 0; 
    $c[$inst]->createText($bw+1+$swXPad-$ti*2, $bw+$neYPad+$yIncr, 
			  -font=>$labelFont, -justify=>'right',
			  -anchor=>'ne', -text=>$$plotYMax[0]);
    $c[$inst]->createText($bw+1+$swXPad-$ti*2, $bw+$neYPad+$yAxis, 
			  -font=>$labelFont,-justify=>'right', 
			  -anchor=>'e', -text=>$$plotYMin[0])
                                if $plotType!~/r/;

    # *** MINOR Y-AXISES ***
    # Addtional Y-AXIS Labels on the right if we have any
    # For now I'm not sure what to do for radial plots
    my $xLoc=$bw+$swXPad+$lineWidth+$xAxis+$lineWidth+1;
    my $yLocTop=$bw+$neYPad+$lineWidth;
    my $yLocBot=$bw+$neYPad+$lineWidth+$yAxis;
    for (my $i=1; $i<$numYLabel; $i++)
    {
      my $minLabel=@$plotref[$plotNum]->{YMinLabel}->[$i];
      my $maxLabel=@$plotref[$plotNum]->{YMaxLabel}->[$i];
      $c[$inst]->createText($xLoc+$ti*4, $yLocTop,
			    -font=>$labelFont, -justify=>'right',
			    -anchor=>'nw', 
			    -text=>$maxLabel)
                                if $plotType!~/r/;
      $c[$inst]->createText($xLoc+$ti*4, $yLocBot,
			    -font=>$labelFont,-justify=>'right',
			    -anchor=>'sw',
			    -text=>$minLabel)
                                if $plotType!~/r/;
      $yLocTop+=$th;
      $yLocBot-=$th;
    }
  }

  # Title above rectangle/circle
  if ($geometry eq 'cd')
  {
    $title=$hostname[$hostNum];
    $c[$inst]->createText($bw+1+$swXPad+$xAxis/2, $bw+$ti*2, 
			  -font=>$labelFont, 
			  -anchor=>'n', -text=>$title);
  }

  # Diagnostics
  if ($debug & 64)
  {
    printf "Rectangle  [%d,%d][%d,%d]\n", $bw+1, $bw+1, $width+$bw, $height+$bw;
    $c[$inst]->create('rectangle',
		      $bw+1, $bw+1, $width+$bw, $height+$bw,
		      -outline=>'blue', -width=>1);
    printf "UpperLeft  [%d,%d][%d,%d]\n", 
                      $bw+$swXPad+1, $bw+$neYPad+1, 
                      $bw+$swXPad+2, $bw+$neYPad+2;
    $c[$inst]->create('rectangle',
		      $bw+$swXPad+1, $bw+$neYPad+1, 
		      $bw+$swXPad+2, $bw+$neYPad+2,
		      -outline=>'white', -width=>1);
    printf "TopMiddle  [%d,%d][%d,%d]\n",
                      $bw+$swXPad+1+($xAxis)/2,   $bw+$neYPad+1,
                      $bw+$swXPad+2+($xAxis)/2, $bw+$neYPad+2;
    $c[$inst]->create('rectangle',
		      $bw+$swXPad+1+($xAxis)/2,   $bw+$neYPad+1, 
		      $bw+$swXPad+2+($xAxis)/2, $bw+$neYPad+2,
		      -outline=>'white', -width=>1);
    printf "LowerRight [%d,%d][%d,%d]\n",
                      $bw+$swXPad+$xAxis,   $bw+$neYPad+$yAxis,
                      $bw+$swXPad+$xAxis+1, $bw+$neYPad+$yAxis+1;
    $c[$inst]->create('rectangle',
		      $bw+$swXPad+$xAxis,   $bw+$neYPad+$yAxis, 
		      $bw+$swXPad+$xAxis+1, $bw+$neYPad+$yAxis+1,
		      -outline=>'white', -width=>1);
  }

  $c[$inst]->pack(-side=>'left');
  $frame2->pack(-side=>'left');
}

sub getData
{
  #    G e t    I n p u t

  # ...but first handle any ^C's
  # Does it really have to be this complicated?  I'm not sure, but it seems
  # that if one wants to do the double ^C thing (which is REALLY handy when
  # debugging screen layouts), one has to get clever with Tk and I've already
  # put far too much time into getting the logic workin to tinker with it again
  if ($ctrlCFlag)
  {
    if ($ctrlCFlag==1)
    {
      print "Ouch!\n";
      print "...takes one more ^C to kill when debugging and not logging\n"    if $debug;
      $socket->close();
      unlink $proxyFile    if defined($proxyFile) && !($debug & 512);
    }

    if ($ctrlCFlag==1)
    {
      exit    if !$debug;
      sleep 86400;
      exit;
    }

    $frame->after(10, \&getData);
    return;
  }

  my $remoteAddr;
  while (my @ready=$select->can_read(1))
  {
    # Data avalable let's see from whom
    foreach my $fileHandle (@ready)
    {
      if ($fileHandle==$socket)
      {
	# Create a new socket and add to selection list so we'll look
	# for data from it
	my $new = $socket->accept;
	$remoteAddr=inet_ntoa((sockaddr_in(getpeername($new)))[1]);
	error("Connection from unknown host '$remoteAddr' - could this be a NAT or Cluster Alias?")
	    if !defined($addrhost{$remoteAddr});
	$sockHandle{$new}=$addrhost{$remoteAddr};
	$select->add($new);
        $numConnections++;
	printf "New socket connection [$numConnections] from Host: %d Addr: %s\n", 
	        $sockHandle{$new}, $remoteAddr
		    if $debug & 1;
      }
      else
      {
        $Record=<$fileHandle>;
	if (!defined($Record))
        {
	  # Remote system broke the connection
	  print "Connect closed\n"    if $debug & 1;
          $select->remove($fileHandle);
	  $fileHandle->close;
          $numConnections--;
          exit    if $numConnections==0;
	  next;
        }
	next    if $Record=~/$IgnoreData/;

	print "INPUT>>> $Record"    if $debug & 2;
	processRecord($sockHandle{$fileHandle});
	last;  #<<<<<<<<<<<<<<<<<<<
	next;
      }
      last; #<<<<<<<<<<<<<<<
    }
    # all 'current' available data processed.  could consider moving
    # inside loop above and possibly even having a counter to limit how
    # much data to read before painting screen.  BUT - if we have that
    # much data we'll probably never be able to keep up and warnings or
    # error messages might be more appropriate.
    last;
  }
  # Be VERY careful!  At one time I called after() in two places and they
  # somehow stepped on each other and screen updates stopped even though
  # socket I/O didn't.
  $frame->after(10, \&getData);
}

sub processRecord
{
  my $hostNum=shift;
  my ($temp, $date, $time, @Values);

  ($temp, $date, $time, @Values)=split(/\s+/, $Record);
  $hostNum=$addrhost{$temp}    if defined($proxy);

  return    if $Record=~/#/;    # now we can ignore comments

  #    U p d a t e    T h e    D i s p l a y    F o r    T h i s    H o s t

  my $addr=$hostaddr[$hostNum];
  my $plotref=$plotinfo{$addr};
  for (my $plotNum=0; $plotNum<$numPlots; $plotNum++)
  {
    # Since we don't necessarily have the same number of plots for all
    # addresses, skip those that are undefined.
    next    if !defined(@$plotref[$plotNum]);

    # Get the instance number of this host for each plot    
    my $inst=$hostNum*$numPlots+$plotNum;

    # Since not all plots for the same host have the same number of data
    # points their sweeper doesn't hit the beginning at the same time and
    # so we have to reset their X position one plot at a time.

    my $numCols=scalar(@{@$plotref[$plotNum]->{YColnum}});
    resetLast($inst, $numCols, $plotref)
	  if ++$curPoint[$inst]>=$maxPoints[$inst];

    # Ok, now update an individual plot, one line at a time
    my ($stackVal, $plotVal, $subval, $lastOper)=(0,0,0,' ');
    my $colNum=0;
    for (my $i=0; $i<$numCols; $i++)
    {
      my $col=@$plotref[$plotNum]->{YColnum}->[$i];
      my $div=@$plotref[$plotNum]->{YDiv}->[$i];
      my $oper=@$plotref[$plotNum]->{Oper}->[$i];
      #print "COL: $col OPER: $oper  DIV: $div  SUBVAL: $subval  ";

      if ($homoFlag && !defined($Values[$col]))
      {
        if ($debug & 1)
        {
  	  print "Can't find value as directed in column $col of data stream for host: $hostNum\n";
          print "Maybe systems are not homogeneous!\n";
        }
        next;
      }

      # Remember, when col==-1, we have a constant stored in 'YNames'
      my $val;
      if ($col>=0)
      {
	$val=$Values[$col]/$div;
      }
      else
      {
        $val=@$plotref[$plotNum]->{YNames}->[$i];
        if ($val=~/(\d+)([KMG])/)
        {
          $val=$1;
	  $val*=1024              if $2 eq 'K';
	  $val*=1024*1024         if $2 eq 'M';
	  $val*=1024*1024*1024    if $2 eq 'G';
        }
      }

      #print "VAL: $val  LAST: $lastOper  ";
      $subval+=$val    if $lastOper eq '+' || $lastOper eq ' ';
      $subval-=$val    if $lastOper eq '-';
      $subval*=$val    if $lastOper eq '*';
      $subval/=$val    if $lastOper eq '/' && $val!=0;
      #print "VAL: $val  SUBVAL: $subval\n";
      if ($oper ne ' ')
      {
        $lastOper=$oper;
	next;
      }

      my $value=$subval;
      #print "VALUE: $value\n";
      $subval=0;
      $lastOper=' ';
      
      $stackVal+=$value;
      $plotVal=(@$plotref[$plotNum]->{PType}=~/[bs]/) ? $stackVal : $value;
      printf "Plot[%d,%d,%d]%s %-7s Val: %7s Divisor: %4d ".
	     "YMin: %4d YMax: %4d PlotVal: %d\n", 
             $hostNum, $plotNum, $colNum,
	     @$plotref[$plotNum]->{PType}=~/[bs]/ ? '*' : '',
             $plotColor[$colNum], $Values[$col], $div, 
             @$plotref[$plotNum]->{YMin}->[$colNum], 
             @$plotref[$plotNum]->{YMax}->[$colNum], $plotVal
		if $debug & 32;

      plotUpdate($inst, $colNum, $plotVal);
      $colNum++;
    }
  }
}

sub plotUpdate
{
  my $inst=   shift;
  my $colNum= shift;
  my $value=  shift;
  my ($time, $xNew, $yNew, $ymin, $ymax);

  exit    if $ctrlCFlag;

  # in general, this routing doesn't care about the number of the plot it's 
  # updating except for the fact that a plot can have multiple Y-scales and 
  # therefore it needs to know the limits of each individual column of data.
  my $hostNum=int($inst/$numPlots);
  my $plotNum=$inst % $numPlots;

  my $addr=$hostaddr[$hostNum];
  my $plotref=$plotinfo{$addr};

  my $tag="pm-$inst:$colNum-$curPoint[$inst]";
  $c[$inst]->delete($tag);

  # adjust values based on smoothing value
  my $vNew=int($smooth*$vLast[$inst][$colNum]+(1-$smooth)*$value);
  print "Val: $value Last: $vLast[$inst][$colNum] Smoothed: $vNew\n"
      if $debug & 32 && $smooth ne 0;

  #    C a l c u l a t e   X / Y    C o o r d i n a t e s

  $ymin=@$plotref[$plotNum]->{YMin}->[$colNum];
  $ymax=@$plotref[$plotNum]->{YMax}->[$colNum];

  my $percent=($vNew-$ymin)/($ymax-$ymin);
  if ($percent<0 || $percent>1)
  {
    $percent=0    if $percent<0;
    $percent=1    if $percent>1;
    my $instName=(@$plotref[$plotNum]->{InstName} ne '') ?
	":@$plotref[$plotNum]->{InstName}" : '';

    # be sure to flag stacked values so we know WHY multiple low-value data
    # points are being rejected!
    printf "Data point $value%s outside range of ".
	  "$ymin-$ymax for plot '@$plotref[$plotNum]->{Name}$instName', field ".
	  "'@$plotref[$plotNum]->{YLabels}->[$colNum]'\n",
	      @$plotref[$plotNum]->{PType}=~/[bs]/ ? '*' : ''
	          if $debug & 1;
  }

  # In dense mode, we've actually made the rectangle small enough to fit the
  # the time info so we need to make an adjustment.  The easiest way to handle
  # this is to track which row (0-based) of the display we're working on.
  my $displayRow=int($inst/$numPerRow);

  my $angle;
  my $plotType=@$plotref[$plotNum]->{PType};
  if ($plotType!~/r/)
  {
    # gotta take lots of pixels into account when calculating Y...
    my $y=$yAxis-$lineWidth;
    my $yPercent=int($y-$y*$percent);
    my $yOffset=$bw+$neYPad+$rectWidth+int($lineWidth/2);
    $yNew=$yOffset+$yPercent;
    $xNew=$xLast[$inst][$colNum]+$plotWidth;
    #printf "y: $y  YPERCENT: $yPercent  YOFF: $yOffset  YNEW: $yNew\n";
  }
  else
  {
    my $size=$radius*$percent;
    $angle=180+($maxPoints[$inst]-$curPoint[$inst])*$plotDegr;
    $xNew=polarX($xCenter, $size-$lineWidth, $angle);#-$lineWidth;
    $yNew=polarY($yCenter, $size-$lineWidth, $angle);#-$lineWidth;
    #print "Angle: $angle  Size: $size  X: $xNew  Y: $yNew\n";
  }

  # We have to set the y position the first time through and in the case of
  # radial plots, we need to set both.  For xyPlots we already have set up x
  # in resetLast().
  if (!defined($yLast[$inst][$colNum]))
  {
    $xLast[$inst][$colNum]=$xNew    if $plotType=~/r/;
    $yLast[$inst][$colNum]=$yNew;
  }

  my $colPos=@$plotref[$plotNum]->{YColnum}->[$colNum];
  $xLast=$xLast[$inst][$colNum];
  $yLast=$yLast[$inst][$colNum];
  printf "Plot[%d,%d,%d] Col: %2d  Val: %5d  YMin: %5d YMax: %5d ".
         "Percent: %3d [$xLast[$inst][$colNum],%s] To [$xNew,$yNew]\n",
      $inst, $plotNum, $colNum, $colPos, $vNew, 
      $ymin, $ymax, $percent*100, defined($yLast) ? $yLast : ''
           if $debug & 4;

  #    U p d a t e    P l o t

  if ($plotType=~/l/)
  {
    $c[$inst]->createLine($xLast, $yLast, $xNew, $yNew, 
			  -tags=>$tag, -fill=>$plotColor[$colNum],  
			  -width=>$lineWidth);
  }
  elsif ($plotType=~/p/)
  {
    # This is a little weird.  By  the nature of the way point plots work,
    # we need to not only print the current data, but since it's in the second
    # position (an artifact of the way the sweeper works), we also need to print
    # the preceeding point.
    $c[$inst]->createRectangle($xLast, $yLast, $xLast, $yLast,
			       -tags=>$tag, 
			       -outline=>$plotColor[$colNum], 
			       -width=>$lineWidth)
	if $curPoint[$inst]==0;

    # same as line except instead of using 'last' coordinate we build a
    # line (or circle?) a whole length is the width of a line.  Real subtle, but
    # for wide lines, we need to back off by 1/2 width.
    $c[$inst]->createRectangle($xNew-$lineWidth/2, $yNew, 
			       $xNew-$lineWidth/2, $yNew,
			       -tags=>$tag, 
			       -outline=>$plotColor[$colNum], 
			       -width=>$lineWidth);
  }
  elsif ($plotType=~/b/)
  {
    # bars always stacked and start at the position of the last high mark
    # the first bar (col==0) starts at the bottom.
    $lastHigh=$bw+$neYPad+$yAxis+$rectWidth    if $colNum==0;

    $c[$inst]->createRectangle($xLast, $lastHigh, $xNew, $yNew, 
			       -tags=>$tag, -fill=>$plotColor[$colNum], 
			       -width=>0);
    $lastHigh=$yNew+1;
  }

  #    U p d a t e    S w e e p e r    &    T i m e

  # only need to update the sweeper/time when the only the first plot line changes
  if ($colNum==0)
  {
    $c[$inst]->delete("sweep");
    if ($plotType!~/r/)
    {
      # In dense mode, we've actually made the rectangle small enough to fit the
      # the time info so we need to make an adjustment.  The easiest way to handle
      # this is to track which row (0-based) of the display we're working on.
      $displayRow=int($inst/$numPerRow);

      # *** SWEEPER HERE ***
      # Remember, rectangle is drawn in middle of width (don't forget -d 321)
      my $sweepStart=$bw+$neYPad+int($rectWidth/2);
      $c[$inst]->createLine($xNew, $sweepStart, 
			    $xNew, $sweepStart+$yAxis+$lineWidth,
			    -tags=>'sweep', -fill=>'black', -width=>1);

      # *** XAXIS TIME HERE ***
      # But remember that in dense mode we only do the first row
      if (($timeFlag) && 
          ($geometry!~/d/ || ($geometry=~/d/ && $displayRow==0)))
      {
        # once counter hits time to draw labels for this host, leave it alone
        # until we draw the last one.  As a minor optimization, we're keeping
        # track of when a full graph is filled so we don't have to redraw
        # time ticks.
	$firstPassFlag=0    if $curPoint[$inst]==$maxPoints[$inst]-1;

	if (($curPoint[$inst] % $tickMinFreq)==0)
	{
	  # Either update the major tick mark (and time) or the minor one
	  # but not both.
	    my $yTick=$bw+$neYPad+$yAxis+$rectWidth*2-1;
	    if (($curPoint[$inst] % $tickMajFreq)==0)
	    {
	      $c[$inst]->createLine(
		    $xLast, $yTick,
		    $xLast, $yTick+$tMajL,
	            -fill=>$plotColor[$tMajC], -width=>$tMajW)
	                  if $firstPassFlag;
	    
	    # time is either not or zero based from start of colgui
	    if (!$timeZeroFlag)
	    {
              $time=localtime(time);
	      $time=(split(/\s+/, $time))[3];
	    }
	    else
	    {
              my ($hours, $mins, $secs);
	      $time=$timeStart-time;
	      $hours=int($time/3600);
	      $mins= int(($time-$hours*3600)/60);
	      $secs= $time-$time-$hours*3600-$time-$mins*60;
	      $time=sprintf("%02d:%02d:%02d", $hours, $mins, $secs);
	    }

	    # *** ACTUAL TIME HERE ***
	    my $markIndex=$curPoint[$inst]/$tickMajFreq;
	    my $timetag="time-$markIndex";
	    $c[$inst]->delete($timetag);
	    $c[$inst]->createText(
      		   $xLast, $yTick+$tMajL+2,
	           -fill=>$plotColor[$tMajC], -anchor=>'n', -font=>$timeFont,
    	           -tag=>$timetag, -text=>$time);
          }
	  elsif ($firstPassFlag)
          {
	    # *** MINOR TICK HERE ***
            $c[$inst]->createLine(
                    $xLast, $yTick,
                    $xLast, $yTick+$tMinL,
                    -fill=>$plotColor[$tMinC], -width=>$tMinW);
	  }
        }
      }
    }
    else
    {
      $c[$inst]->createLine($xCenter, $yCenter, 
			  polarX($xCenter, $radius-$lineWidth, $angle),
			  polarY($yCenter, $radius-$lineWidth, $angle),
			  -tags=>'sweep', -fill=>'black', -width=>$lineWidth);
    }
  }

  $yLast[$inst][$colNum]=$yNew;
  $xLast[$inst][$colNum]=$xNew;
  $vLast[$inst][$colNum]=$vNew;
}

# resets 'last' variables for all individual plots within 'inst'
sub resetLast
{
  my $inst=   shift;
  my $cols=   shift;
  my $plotref=shift;
  my $col;

  my $plotNum=$inst % $numPlots;
  my $plotType=@$plotref[$plotNum]->{PType};

  for ($col=0; $col<$cols; $col++)
  {
    # Note that we initialize Y elsewhere in a plot specific manner and
    # then only one time.  Also note we NEVER reset the x values for radial.
    # since the first thing we do to X is increment it by the width of the
    # plot width, we need to set it negative!  Finally, I'm still not sure
    # why -2, but it works and I'm afraid to change it.
    $xLast[$inst][$col]=$bw+$swXPad+$rectWidth    if $plotType!~/r/;
    $vLast[$inst][$col]=$vLast[$inst][$col]=0;
  }
  $curPoint[$inst]=0;
}

# Thanks to Donagh McCabe for these...
sub polarX
{
  my ($x, $r, $theta)=@_;

  my $thetaRadian=$theta/57.2957795;
  my $xFromZero=$r*cos($thetaRadian);
  my $result=$x+int($xFromZero+.5);
  #print "x: $x, r: $r, t: $theta  res: $result\n";

  return($result)
}

sub polarY
{
  my ($y, $r, $theta)=@_;

  my $thetaRadian=$theta/57.2957795;
  my $yFromZero=$r*sin($thetaRadian);
  my $result=$y-int($yFromZero+.5);
  #print "y: $y, r: $r, t: $theta  res: $result\n";

  return($result);
}

sub toTime
{
  my $secs=shift;

  my $hour=int($secs/3600);
  my $min= int(($secs-$hour*3600)/60);
  my $sec=$secs-$hour*3600-$min*60;
  return(sprintf("%d:%02d:%02d", $hour, $min, $sec));
}

# Initializes @hostname, @hostnameV and @hostnum.  Returns $numHosts
# This is really for both machines as well as /etc/hosts
sub readMachines
{
  my $machines=shift;
  my $grep=    shift;
  my ($command, $numHosts, $line, $addr, $host, $hostV, $address);

  $numHosts=0;
  $command=(!defined($grep)) ? "cat $machines" : "grep $grep $machines";
  print "Select machines: $command\n"    if $debug & 1;
  open MAC, "$command|" or error("Couldn't execute '$command'");
  foreach $line (<MAC>)
  {
    chomp $line;
    next    if $line=~/^#|^\s*$/;
    ($addr, $host, $hostV)=split(/\s+/, $line);

    $host=$addr     if !defined($host);
    $hostV=$host    if !defined($hostV);

    $hostaddr[$numHosts]=$addr;
    $hostname[$numHosts]=$host;
    $address=inet_ntoa(scalar(gethostbyname($addr)));
    $addrhost{$address}=$numHosts;

    # if it exists, use the 3rd column of the machines file for displaying
    # the machine name vertically when -gnc is used
    $hostV=$host    if !defined($hostV);
    $hostV=~s/(.{1})/$1\n/g;
    $hostV=~s/\n$//;
    $hostnameV[$numHosts]=$hostV;

    $numHosts++;
  }
  close MAC;
  return($numHosts);
}

# is there a better way?
sub getReturnAddress
{
  my $address=shift;
  my ($line, $destaddr, $gateway, $mask, $interface, $octet, $myaddr);
  my (@addrOctets, @destOctets, @maskOctets);

  return($realAddress)    if defined($realAddress);

  # in case hostname, convert back to an address and cry if we can't
  print "Get return address associated with $address\n"    if $debug & 1;
  my $gbn=gethostbyname($address);
  error("cannot resolve $address to a network address")    if $gbn eq '';
  $address=inet_ntoa(scalar($gbn));

  @addrOctets=split(/\./, $address);
  open ROUTES, "$Route|" or error("Couldn't execute '$Route'");
  foreach $line (<ROUTES>)
  {
    next    if $line!~/^\d|^default/;
    chomp $line;

    ($destaddr, $gateway, $mask, $interface)=(split(/\s+/, $line))[0,1,2,7];

    # Note if default route we don't have any digits in here, but since the
    # mask is 0.0.0.0 if will kick on on the first test.
    @destOctets=split(/\./, $destaddr);
    @maskOctets=split(/\./, $mask);
    for (my $i=0; $i<4; $i++)
    {
      # we're guaranteed a hit since the default starts with 0.
      if ($maskOctets[$i]==0)
      {
        close ROUTES;
	$myaddr=`$Ifconfig $interface | grep addr:`;
	$myaddr=(split(/\s+/, $myaddr))[2];
	$myaddr=(split(/:/, $myaddr))[1];
        print "Got address: $myaddr\n"    if $debug & 1;
	return ($myaddr);
      }
      last    if ($addrOctets[$i] & $maskOctets[$i])!=$destOctets[$i];
    }
  }

  # The only way to make sure this never happens is to put the code
  # in to catch it.
  error("Can't find default route in $Route");
}

sub sigInt
{
  $ctrlCFlag++;
}

sub sigPipe
{
  print "Broken Pipe!!!\n";
  exit;
}

sub error
{
  print "colgui: $_[0]\n";
  exit;
}

sub getRemoteConfig
{
  my $cfgref= shift;
  my $globals=shift;

  my $result;
  my $errors=0;
  my $speedmax=0;
  my $addr0=$hostaddr[0];    # address of 1st host
  my $allLustypes='';
  my $remote=(defined($rsh)) ? 'rsh ' : 'ssh ';
  for (my $hostIdx=0; $hostIdx<scalar(@hostaddr); $hostIdx++)
  {
    my $addr=$hostaddr[$hostIdx];
    print "  $addr\n";

    if ($homoFlag && $hostIdx>0)
    {
      # In homogenous mode, we use identical configurations for everything.
      $cfgref->{$addr}->{colver}=$cfgref->{$addr0}->{colver};
      $cfgref->{$addr}->{memory}=$cfgref->{$addr0}->{memory};
      next;
    }

    # Only if we expect to directly reach them
    if (!defined($proxy))
    {
      $result=`ping -c1 $addr`;
      $result=~/(\d+) received/;
      if ($1==0)
      {
        print "    could not ping host '$addr' so it will not be monitored\n";
	$hostaddr[$hostIdx]=0;
	$errors++;
	next;
      }
    }

    my $commandBegin=(!defined($proxy)) ?
	"$remote $username$addr $colbin" : 
	"$remote $username$proxy \"$colmuxbin -addr $addr -port $port -command '";
    my $switches='-v';
    my $command="$commandBegin $switches";
    $command.="'\""                if defined($proxy);  # need to close quotes
    print "Command: $command\n"    if $debug & 1;

    # We're doing the command this way because not all collectl messages go
    # to stderr.  The only one that's good is the version output.
    `$command 1>$tmpFile 2>&1`;
    $result=`cat $tmpFile`;
    $result=~/collectl V(\S+)/;
    print "Result:  $result"    if $debug & 8;
    if ($result!~/collectl V(\S+)/s)
    {
      printf "    expected 'Version' not '%s'\n", $result;
      $hostaddr[$hostIdx]=0;
      $errors++;
      next;
    }
      
    $cfgref->{$addr}->{colver}=$1;
    if ($cfgref->{$addr}->{colver} lt '2.1.1')
    {
      printf "    collectl V2.1.1 or greater required\n";
      $hostaddr[$hostIdx]=0;
      $errors++;
      next;
    }

    #    R e q u e s t    C o n f i g u r a t i o n

    # We want to get the sytem configuration from the header, but
    # unfortunately, collectl doesn't report interconnect or lustre
    # info unless explicitly requested, so force the issue below.
    $switches= "-s+lx --lustsvc com --showheader";
    $command="$commandBegin $switches";
    $command.="'\""             if defined($proxy);
    print "Command: $command\n"    if $debug & 1;

    # Make sure return at least contains part of a header OR an error
    `$command 1>$tmpFile 2>&1`;
    $result=`cat $tmpFile`;
    if ($result=~/Error: (.*)/)
    {
      print "    Collectl on node $addr reported error '$1'\n";
      $errors++;
      next;
    }
    elsif ($result!~/Collectl:/s)
    {
      printf "   expected line containing 'Collect' but got: $result";
      $hostaddr[$hostIdx]=0;
      $errors++;
      next;
    }

    $result=~/Memory:\s+(\d+)/;
    $cfgref->{$addr}->{memory}=int($1/1024);

    $result=~/NetNames: (.*)/;
    my $netNames=$1;
    foreach my $net (split(/\s+/, $netNames))
    {
      my $speed=(split(/:/, $net))[1];
      $speedmax=$speed    if defined($speed) && $speed ne '??' && $speed>$speedmax;
    }

    my $lusCltNames=($result=~/LustreClient:\s+CltInfo:\s+(.*)/) ? $1 : '';
    my @temp=split(/\s+/, $lusCltNames);
    my $numLusClt=@temp;
    my $numLusMds=($result=~/NumMds:\s+(\d+)\s/) ? $1 : 0;
    my $numLusOst=($result=~/NumOst:\s+(\d+)\s/) ? $1 : 0;

    my $thisLustype='';
    $thisLustype.='c'    if $numLusClt;
    $thisLustype.='m'    if $numLusMds;
    $thisLustype.='o'    if $numLusOst;
    $allLustypes.='c'    if $numLusClt && $allLustypes!~/c/;
    $allLustypes.='m'    if $numLusMds && $allLustypes!~/m/;
    $allLustypes.='o'    if $numLusOst && $allLustypes!~/o/;

    printf "Lustre Clt: %d  Mds: %d  Ost: %d  ThisType: %s  AllTypes: %s\n", 
        $numLusClt, $numLusMds, $numLusOst, $thisLustype, $allLustypes
	    if $debug & 8;

    if ($subsys=~/l/i && $thisLustype eq '')
    {
      print "    this node does not support lustre\n";
      $errors++;
      next;
    }

    $cfgref->{$addr}->{lustype}=$thisLustype;
  }

  $globals->{alltypes}=$allLustypes;
  $globals->{speedmax}=$speedmax;
  return($errors);
}

sub getRemoteHeader
{
  my $cfgref= shift;
  my $subsys= shift;
  my $lusopt= shift;

  my $thisLustOpts='';
  my $addr0=$hostaddr[0];
  my $remote=(defined($rsh)) ? 'rsh ' : 'ssh ';
  for (my $hostIdx=0; $hostIdx<scalar(@hostaddr); $hostIdx++)
  {
    my $addr=$hostaddr[$hostIdx];
    if ($homoFlag && $hostIdx>0)
    {
      $cfgref->{$addr}->{header}=$cfgref->{$addr0}->{header};
      next;
    }

    my $commandBegin=(!defined($proxy)) ?
	"$remote $username$addr $colbin" : 
	"$remote $username$proxy \"$colmuxbin -addr $addr -port $port -command '";

    # we're now request the headers (a sampling interval of 0 and ignore
    # data) based on the plots requested.
    # or intereconnects.
    my $lustype=$cfgref->{$addr}->{lustype};
    $thisLustOpts='MR' if $lustype eq 'c';
    $thisLustOpts='B'  if $lustype eq 'o';
    $thisLustOpts=''   if $lustype eq 'm';

    $switches= "-s $subsys -P -i0 -c1";
    $switches.=" --lustsvc  $lustype"         if $lustype ne '';
    $switches.=" --lustopts $thisLustOpts"    if $thisLustOpts ne '';
    $command="$commandBegin $switches";
    $command.="'\""                  if defined($proxy);
    print "Command: $command\n"      if $debug & 1;

    my $result=`$command`;
    $result=~/(#.*)/;
    $cfgref->{$addr}->{header}=$1;
  }
}

# Stolen from colplot, noting we're now using '$config' instead of '$paramsConfig'
sub showplots
{
  my $mycfg=shift;

  my $config={};
  initParamsInit($mycfg, $config);

  # Being lazy, let's sort the descriptions by category, modifier and type this way.
  my %sorted;
  my $numPlots=0;
  foreach my $plotname (keys %{$config->{PlotDesc}})
  {
    my $cat= $config->{PlotDesc}->{$plotname}->{cat};
    my $type=$config->{PlotDesc}->{$plotname}->{type};
    my $mod= (defined($config->{PlotDesc}->{$plotname}->{mod})) ?
                $config->{PlotDesc}->{$plotname}->{mod} : '';

    # temporarily change the type of detail and macros to force their sort
    $type='y'    if $type eq 'd';
    $type='z'    if $type eq 'm';
    $sorted{"$type:$cat:$mod:$plotname"}='';
    $numPlots++;
  }

  # This is for terminal-based output
  foreach my $key (sort keys %sorted)
  {
    my ($type, $cat, $mod, $plotname)=split(/:/, $key);

    $type='d'    if $type eq 'y';   # details
    $type=''     if $type eq 'z';   # macros

    printf "%-11s %-6s  %1s  %s\n",
          $plotname, $cat, $type, $config->{PlotDesc}->{$plotname}->{desc};
  }
  exit;
}

sub usagev
{
  print "$mycfg->{exebare}-$Version\n";
  exit;
}

sub help
{
print <<EOF;
usage: colgui [--address|--hosts|--machines] -switches --switches

To get started, all you need are the first set of switches immediately below. 
Once you start to feel more comfortable start to experiment with the 
'Plotting/Layout' switches.

Common Options & Plot Selection

    -i int           data collection interval, same as collectl -i, [def=1]
    -r size          number of plots per row, [def=3]
    -s subs          sub-systems to plot, same as collectl -s, [def=cm]
    --filter  list   list of strings that will be applied against any detail
                     plots selected via uppercase subsystems.  only those that
		     match will be selected.
    
Host Selection - default is local host

    --address    addr      address to connect to, instead of using --machines
    --hosts      pattern   /etc/hosts is grep'd with this pattern
    --machines   machines  name of file containing addresses to connect to

Alternative Plot Selection, can be combined with -s
    -p plots  names(s) of standard plots, some already used by -s
              --showplots to see list of valid values

Different Types of Help
    -h                  print this help text
    -x                  help on more advanced topics
    -v                  print version and exit
    --showaddrs         list addresses of hosts selected for processing
    --showparams        print plotting parameters based on selections and exit
    --showplots         show list of valid plots (for use with -p)

$Copyright
EOF
exit;
}

sub helpAdvanced
{
print <<EOF2;
Advanded plotting options

  Most Common
    --xaxis    int   size of x-axis in number of intervals (see -i)
    --yaxis    pix   size of y-axis in pixels.  also used for diameter of radial plots
    --geometry geo   display using one of the following geometries
                         n -  normal, each host starts a new row, this is the default
                         c -  compressed, plots fill rows ignoring host names
                         nd - normal and dense, redundant information factored out
			 cd - compressed and dense

  Less Common
    --homogeneous      assume all nodes have IDENTICAL configurations, 
                       speeding up startup
    --plottype   type  for all plots to display in this format
                         b - bar           s - stacked
                         l - line          r - radial
                         p - point
    --radint     num   number of intervals in a radial plot, [def=--xaxis]
    --smooth     S     apply smoothing value 'S, where 0<S<1 to data using the equation
                         newVal = S*oldVal + (1-S)*currentVal

    --linewidth  num   number of pixels in plot line, [def=1]
    --plotwidth  num   number of pixels in 1 plot interval, [def=3]

Data Collection and Logging Options:

    --count      num     number of samples to collect
    --colbin     bin     Path to collectl executable [def=/usr/sbin/collect]
    --colmuxbin  bin     Path to proxy executable [def=/usr/sbin/colmux]

    --lustype    opts    Select types of lustre nodes to report on
                           c - client,  o - oss,  m - mds  [def=o]
    --nsftype    opts    Type of nfs (may be combined) [def=V3,server]
                           c - client, 2 - version2

    --logdir     dir     Tell collectl to log all data to dir/
                           note that in proxy mode the existence of dir
                           cannot be verified
Network/Communications options:

    --port       port    Port remote collectl should use for communications
    --proxy      addr    Address of colgui-proxy
    --realaddr   addr    Use THIS address (typically hidden by NAT) for communications
    --rsh                Use 'rsh' instead of 'ssh' for communictions
    --username   name    Username to use in rsh/ssh command to remote machine
EOF2
exit;
}
collectl-utils 4.7.1-1 / usr / bin / colgui
