percona-toolkit 2.2.16-1 / usr / bin / pt-sift

#!/usr/bin/env bash

# This program is part of Percona Toolkit: http://www.percona.com/software/
# See "COPYRIGHT, LICENSE, AND WARRANTY" at the end of this file for legal
# notices and disclaimers.

# ###########################################################################
# log_warn_die package
# This package is a copy without comments from the original.  The original
# with comments and its test file can be found in the Bazaar repository at,
#   lib/bash/log_warn_die.sh
#   t/lib/bash/log_warn_die.sh
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################


set -u

PTFUNCNAME=""
PTDEBUG="${PTDEBUG:-""}"
EXIT_STATUS=0

ts() {
   TS=$(date +%F-%T | tr ':-' '_')
   echo "$TS $*"
}

info() {
   [ ${OPT_VERBOSE:-3} -ge 3 ] && ts "$*"
}

log() {
   [ ${OPT_VERBOSE:-3} -ge 2 ] && ts "$*"
}

warn() {
   [ ${OPT_VERBOSE:-3} -ge 1 ] && ts "$*" >&2
   EXIT_STATUS=1
}

die() {
   ts "$*" >&2
   EXIT_STATUS=1
   exit 1
}

_d () {
   [ "$PTDEBUG" ] && echo "# $PTFUNCNAME: $(ts "$*")" >&2
}

# ###########################################################################
# End log_warn_die package
# ###########################################################################

# ###########################################################################
# tmpdir package
# This package is a copy without comments from the original.  The original
# with comments and its test file can be found in the Bazaar repository at,
#   lib/bash/tmpdir.sh
#   t/lib/bash/tmpdir.sh
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################


set -u

PT_TMPDIR=""

mk_tmpdir() {
   local dir="${1:-""}"

   if [ -n "$dir" ]; then
      if [ ! -d "$dir" ]; then
         mkdir "$dir" || die "Cannot make tmpdir $dir"
      fi
      PT_TMPDIR="$dir"
   else
      local tool="${0##*/}"
      local pid="$$"
      PT_TMPDIR=`mktemp -d -t "${tool}.${pid}.XXXXXX"` \
         || die "Cannot make secure tmpdir"
   fi
}

rm_tmpdir() {
   if [ -n "$PT_TMPDIR" ] && [ -d "$PT_TMPDIR" ]; then
      rm -rf "$PT_TMPDIR"
   fi
   PT_TMPDIR=""
}

# ###########################################################################
# End tmpdir package
# ###########################################################################

# ###########################################################################
# parse_options package
# This package is a copy without comments from the original.  The original
# with comments and its test file can be found in the Bazaar repository at,
#   lib/bash/parse_options.sh
#   t/lib/bash/parse_options.sh
# See https://launchpad.net/percona-toolkit for more information.
# ###########################################################################





set -u

ARGV=""           # Non-option args (probably input files)
EXT_ARGV=""       # Everything after -- (args for an external command)
HAVE_EXT_ARGV=""  # Got --, everything else is put into EXT_ARGV
OPT_ERRS=0        # How many command line option errors
OPT_VERSION=""    # If --version was specified
OPT_HELP=""       # If --help was specified
PO_DIR=""         # Directory with program option spec files

usage() {
   local file="$1"

   local usage="$(grep '^Usage: ' "$file")"
   echo $usage
   echo
   echo "For more information, 'man $TOOL' or 'perldoc $file'."
}

usage_or_errors() {
   local file="$1"
   local version=""

   if [ "$OPT_VERSION" ]; then
      version=$(grep '^pt-[^ ]\+ [0-9]' "$file")
      echo "$version"
      return 1
   fi

   if [ "$OPT_HELP" ]; then
      usage "$file"
      echo
      echo "Command line options:"
      echo
      perl -e '
         use strict;
         use warnings FATAL => qw(all);
         my $lcol = 20;         # Allow this much space for option names.
         my $rcol = 80 - $lcol; # The terminal is assumed to be 80 chars wide.
         my $name;
         while ( <> ) {
            my $line = $_;
            chomp $line;
            if ( $line =~ s/^long:/  --/ ) {
               $name = $line;
            }
            elsif ( $line =~ s/^desc:// ) {
               $line =~ s/ +$//mg;
               my @lines = grep { $_      }
                           $line =~ m/(.{0,$rcol})(?:\s+|\Z)/g;
               if ( length($name) >= $lcol ) {
                  print $name, "\n", (q{ } x $lcol);
               }
               else {
                  printf "%-${lcol}s", $name;
               }
               print join("\n" . (q{ } x $lcol), @lines);
               print "\n";
            }
         }
      ' "$PO_DIR"/*
      echo
      echo "Options and values after processing arguments:"
      echo
      (
         cd "$PO_DIR"
         for opt in *; do
            local varname="OPT_$(echo "$opt" | tr a-z- A-Z_)"
            eval local varvalue=\$$varname
            if ! grep -q "type:" "$PO_DIR/$opt" >/dev/null; then
               if [ "$varvalue" -a "$varvalue" = "yes" ];
                  then varvalue="TRUE"
               else
                  varvalue="FALSE"
               fi
            fi
            printf -- "  --%-30s %s" "$opt" "${varvalue:-(No value)}"
            echo
         done
      )
      return 1
   fi

   if [ $OPT_ERRS -gt 0 ]; then
      echo
      usage "$file"
      return 1
   fi

   return 0
}

option_error() {
   local err="$1"
   OPT_ERRS=$(($OPT_ERRS + 1))
   echo "$err" >&2
}

parse_options() {
   local file="$1"
   shift

   ARGV=""
   EXT_ARGV=""
   HAVE_EXT_ARGV=""
   OPT_ERRS=0
   OPT_VERSION=""
   OPT_HELP=""
   PO_DIR="$PT_TMPDIR/po"

   if [ ! -d "$PO_DIR" ]; then
      mkdir "$PO_DIR"
      if [ $? -ne 0 ]; then
         echo "Cannot mkdir $PO_DIR" >&2
         exit 1
      fi
   fi

   rm -rf "$PO_DIR"/*
   if [ $? -ne 0 ]; then
      echo "Cannot rm -rf $PO_DIR/*" >&2
      exit 1
   fi

   _parse_pod "$file"  # Parse POD into program option (po) spec files
   _eval_po            # Eval po into existence with default values

   if [ $# -ge 2 ] &&  [ "$1" = "--config" ]; then
      shift  # --config
      local user_config_files="$1"
      shift  # that ^
      local IFS=","
      for user_config_file in $user_config_files; do
         _parse_config_files "$user_config_file"
      done
   else
      _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf"
      if [ "${HOME:-}" ]; then
         _parse_config_files "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf"
      fi
   fi

   _parse_command_line "${@:-""}"
}

_parse_pod() {
   local file="$1"

   PO_FILE="$file" PO_DIR="$PO_DIR" perl -e '
      $/ = "";
      my $file = $ENV{PO_FILE};
      open my $fh, "<", $file or die "Cannot open $file: $!";
      while ( defined(my $para = <$fh>) ) {
         next unless $para =~ m/^=head1 OPTIONS/;
         while ( defined(my $para = <$fh>) ) {
            last if $para =~ m/^=head1/;
            chomp;
            if ( $para =~ m/^=item --(\S+)/ ) {
               my $opt  = $1;
               my $file = "$ENV{PO_DIR}/$opt";
               open my $opt_fh, ">", $file or die "Cannot open $file: $!";
               print $opt_fh "long:$opt\n";
               $para = <$fh>;
               chomp;
               if ( $para =~ m/^[a-z ]+:/ ) {
                  map {
                     chomp;
                     my ($attrib, $val) = split(/: /, $_);
                     print $opt_fh "$attrib:$val\n";
                  } split(/; /, $para);
                  $para = <$fh>;
                  chomp;
               }
               my ($desc) = $para =~ m/^([^?.]+)/;
               print $opt_fh "desc:$desc.\n";
               close $opt_fh;
            }
         }
         last;
      }
   '
}

_eval_po() {
   local IFS=":"
   for opt_spec in "$PO_DIR"/*; do
      local opt=""
      local default_val=""
      local neg=0
      local size=0
      while read key val; do
         case "$key" in
            long)
               opt=$(echo $val | sed 's/-/_/g' | tr '[:lower:]' '[:upper:]')
               ;;
            default)
               default_val="$val"
               ;;
            "short form")
               ;;
            type)
               [ "$val" = "size" ] && size=1
               ;;
            desc)
               ;;
            negatable)
               if [ "$val" = "yes" ]; then
                  neg=1
               fi
               ;;
            *)
               echo "Invalid attribute in $opt_spec: $line" >&2
               exit 1
         esac 
      done < "$opt_spec"

      if [ -z "$opt" ]; then
         echo "No long attribute in option spec $opt_spec" >&2
         exit 1
      fi

      if [ $neg -eq 1 ]; then
         if [ -z "$default_val" ] || [ "$default_val" != "yes" ]; then
            echo "Option $opt_spec is negatable but not default: yes" >&2
            exit 1
         fi
      fi

      if [ $size -eq 1 -a -n "$default_val" ]; then
         default_val=$(size_to_bytes $default_val)
      fi

      eval "OPT_${opt}"="$default_val"
   done
}

_parse_config_files() {

   for config_file in "${@:-""}"; do
      test -f "$config_file" || continue

      while read config_opt; do

         echo "$config_opt" | grep '^[ ]*[^#]' >/dev/null 2>&1 || continue

         config_opt="$(echo "$config_opt" | sed -e 's/^ *//g' -e 's/ *$//g' -e 's/[ ]*=[ ]*/=/' -e 's/[ ]*#.*$//')"

         [ "$config_opt" = "" ] && continue

         echo "$config_opt" | grep -v 'version-check' >/dev/null 2>&1 || continue

         if ! [ "$HAVE_EXT_ARGV" ]; then
            config_opt="--$config_opt"
         fi

         _parse_command_line "$config_opt"

      done < "$config_file"

      HAVE_EXT_ARGV=""  # reset for each file

   done
}

_parse_command_line() {
   local opt=""
   local val=""
   local next_opt_is_val=""
   local opt_is_ok=""
   local opt_is_negated=""
   local real_opt=""
   local required_arg=""
   local spec=""

   for opt in "${@:-""}"; do
      if [ "$opt" = "--" -o "$opt" = "----" ]; then
         HAVE_EXT_ARGV=1
         continue
      fi
      if [ "$HAVE_EXT_ARGV" ]; then
         if [ "$EXT_ARGV" ]; then
            EXT_ARGV="$EXT_ARGV $opt"
         else
            EXT_ARGV="$opt"
         fi
         continue
      fi

      if [ "$next_opt_is_val" ]; then
         next_opt_is_val=""
         if [ $# -eq 0 ] || [ $(expr "$opt" : "\-") -eq 1 ]; then
            option_error "$real_opt requires a $required_arg argument"
            continue
         fi
         val="$opt"
         opt_is_ok=1
      else
         if [ $(expr "$opt" : "\-") -eq 0 ]; then
            if [ -z "$ARGV" ]; then
               ARGV="$opt"
            else
               ARGV="$ARGV $opt"
            fi
            continue
         fi

         real_opt="$opt"

         if $(echo $opt | grep '^--no[^-]' >/dev/null); then
            local base_opt=$(echo $opt | sed 's/^--no//')
            if [ -f "$PT_TMPDIR/po/$base_opt" ]; then
               opt_is_negated=1
               opt="$base_opt"
            else
               opt_is_negated=""
               opt=$(echo $opt | sed 's/^-*//')
            fi
         else
            if $(echo $opt | grep '^--no-' >/dev/null); then
               opt_is_negated=1
               opt=$(echo $opt | sed 's/^--no-//')
            else
               opt_is_negated=""
               opt=$(echo $opt | sed 's/^-*//')
            fi
         fi

         if $(echo $opt | grep '^[a-z-][a-z-]*=' >/dev/null 2>&1); then
            val="$(echo $opt | awk -F= '{print $2}')"
            opt="$(echo $opt | awk -F= '{print $1}')"
         fi

         if [ -f "$PT_TMPDIR/po/$opt" ]; then
            spec="$PT_TMPDIR/po/$opt"
         else
            spec=$(grep "^short form:-$opt\$" "$PT_TMPDIR"/po/* | cut -d ':' -f 1)
            if [ -z "$spec"  ]; then
               option_error "Unknown option: $real_opt"
               continue
            fi
         fi

         required_arg=$(cat "$spec" | awk -F: '/^type:/{print $2}')
         if [ "$required_arg" ]; then
            if [ "$val" ]; then
               opt_is_ok=1
            else
               next_opt_is_val=1
            fi
         else
            if [ "$val" ]; then
               option_error "Option $real_opt does not take a value"
               continue
            fi 
            if [ "$opt_is_negated" ]; then
               val=""
            else
               val="yes"
            fi
            opt_is_ok=1
         fi
      fi

      if [ "$opt_is_ok" ]; then
         opt=$(cat "$spec" | grep '^long:' | cut -d':' -f2 | sed 's/-/_/g' | tr '[:lower:]' '[:upper:]')

         if grep "^type:size" "$spec" >/dev/null; then
            val=$(size_to_bytes $val)
         fi

         eval "OPT_$opt"="'$val'"

         opt=""
         val=""
         next_opt_is_val=""
         opt_is_ok=""
         opt_is_negated=""
         real_opt=""
         required_arg=""
         spec=""
      fi
   done
}

size_to_bytes() {
   local size="$1"
   echo $size | perl -ne '%f=(B=>1, K=>1_024, M=>1_048_576, G=>1_073_741_824, T=>1_099_511_627_776); m/^(\d+)([kMGT])?/i; print $1 * $f{uc($2 || "B")};'
}

# ###########################################################################
# End parse_options package
# ###########################################################################

# ###########################################################################
# Global variables
# ###########################################################################

TOOL="pt-sift"
if [ -d "/var/lib/pt-stalk" ]; then
   BASEDIR="/var/lib/pt-stalk"
else
   BASEDIR="$PWD"
fi
PREFIX=""

# ###########################################################################
# Subroutines
# ###########################################################################

sigtrap() {
   echo "Caught signal, exiting" >&2
   rm_tmpdir
   exit 0
}

# Show current help and settings
print_help() {
   cat <<-HELP
   You can control this program with key presses.
                  ---  COMMANDS  ---
      1  Default action: summarize files
      0  Minimal action: list files
      *  View all the files in less
      d  Invoke 'diskstats' on the disk performance data
      i  View the first INNODB STATUS sample in 'less'
      m  Invoke 'pt-mext' to show the SHOW STATUS counters side by side
      n  Summarize the 'netstat -antp' status data
                  --- NAVIGATION ---
      j  Select the next timestamp
      k  Select the previous timestamp
      q  Quit the program
	HELP
}

# ###########################################################################
# Main program loop, called below if tool is ran from the command line.
# ###########################################################################

main() {
   trap sigtrap SIGHUP SIGINT SIGTERM

   # If there's a command-line arg, figure out if it's a file, directory, or
   # prefix.  The outcome of this block of code should be that BASEDIR is the
   # directory where the files live, without a trailing slash; and PREFIX is
   # either empty or a timestamp, such as "2011_02_08_16_58_07".
   if [ $# -eq 1 ]; then
      if [ -d "$1" ]; then
         BASEDIR="$1"
         PREFIX=""
      elif [ -f "$1" -o -f "$1-output" -o -f "$1output" ]; then
         BASEDIR="$(dirname "$1")"
         PREFIX="$(echo "$1" | perl -ne '$_ =~ m/([\d_]+)/; print $1;')"
      else
         echo "Error: $1 is not a directory, and there are no pt-stalk files in the curent working directory ($BASEDIR) with a $1 prefix." >&2
         echo "For more information, 'man pt-sift' or 'perldoc $0'." >&2
         exit 1
      fi
   fi

   # If the programs we need don't exist, try to get them.
   # Percona Toolkit tools:
   for prog in pt-diskstats pt-pmp pt-mext pt-align; do
      # A var can't be named "PR_pt-pmp" so we chop of "pt-" to get
      # the program's basename, resulting in "PR_pmp".
      prog_base=${prog#"pt-"}
      if which "$prog" >/dev/null 2>&1 ; then
         eval "PR_$prog_base"="$(which "$prog")"
      elif [ -f "$prog" -a -x "$prog" ]; then
         eval "PR_$prog_base"="./$prog"
      elif [ -f "${BASEDIR}/$prog" -a -x "${BASEDIR}/$prog" ]; then
         eval "PR_$prog_base"="${BASEDIR}/$prog"
      elif which "curl" >/dev/null 2>&1; then
         echo "Fetching $prog" >&2
         curl -L "https://www.percona.com/get/$prog" > "$prog" && chmod +x "$prog"
         eval "PR_$prog_base"="./$prog"
      else
         echo "Cannot find or fetch required program: $prog" >&2
         exit 1
      fi 
   done

   # We need to generate a list of timestamps, and ask the user to choose one if
   # there is no PREFIX yet.  NOTE: we rely on the "-output" files here.
   (
      cd "$BASEDIR"
      ls *-output 2>/dev/null | cut -d- -f1 | sort > "$PT_TMPDIR/pt-sift.prefixes" 
   )
   if [ ! -s "$PT_TMPDIR/pt-sift.prefixes" ]; then
      echo "Error: There are no pt-stalk files in $BASEDIR" >&2
      echo "For more information, 'man pt-sift' or 'perldoc $0'." >&2
      exit 1
   fi
   if [ -z "${PREFIX}" ]; then
      if [ "$(grep -c . $PT_TMPDIR/pt-sift.prefixes)" = "1" ]; then
         # If there is only one sample, we use it as the prefix.
         PREFIX="$(cat $PT_TMPDIR/pt-sift.prefixes)"
      fi
   fi
   if [ -z "${PREFIX}" ]; then
      echo
      i=0
      cat $PT_TMPDIR/pt-sift.prefixes | while read line; do
         i=$(($i + 1))
         echo -n "  $line"
         if [ $i -eq 3 ]; then
            echo
            i=0
         fi
      done
      # We might have ended mid-line or we might have printed a newline; print a
      # newline if required to end the list of timestamp prefixes.
      awk 'BEGIN { i = 0 } { i++ } END { if ( i % 3 != 0 ) { print "" } }' $PT_TMPDIR/pt-sift.prefixes
      echo
      while [ -z "${PREFIX}" -o "$(grep -c "${PREFIX}" $PT_TMPDIR/pt-sift.prefixes)" -ne 1 ]; do
         DEFAULT="$(tail -1 $PT_TMPDIR/pt-sift.prefixes)"
         read -e -p "Select a timestamp from the list [${DEFAULT}] " ARG
         ARG="${ARG:-${DEFAULT}}"
         if [ "$(grep -c "${ARG}" $PT_TMPDIR/pt-sift.prefixes)" -eq 1 ]; then
            PREFIX="$(grep "${ARG}" $PT_TMPDIR/pt-sift.prefixes)"
         fi
      done
   fi

   KEY=""
   ACTION="DEFAULT"
   while [ "${KEY}" != "q" ]; do

      if [ "${ACTION}" != "INVALID" ]; then
         # Print the current host, timestamp and action.  Figure out if we're at
         # the first or last sample, to make it easy to navigate.
         PAGE="$(awk "/./{i++} /${PREFIX}/{c=i} END{print c, \"of\", i}" $PT_TMPDIR/pt-sift.prefixes)"
         HOST="$(cat "${BASEDIR}/${PREFIX}-hostname" 2>/dev/null)"
         echo -e "======== ${HOST:-unknown} at \033[34m${PREFIX} \033[31m${ACTION}\033[0m (${PAGE}) ========"
      fi

      # Take an action based on the current $ACTION
      case "${ACTION}" in

         # Format a brief report: busiest device's disk stats, CPU stats
         DEFAULT)
            echo "--diskstats--"
            if [ -f "${BASEDIR}/${PREFIX}-diskstats" ]; then

               $PR_diskstats --group-by disk "${BASEDIR}/${PREFIX}-diskstats" \
                  | awk '
                     /ts/ { header = $0 }
                     /[0-9]/ {
                        io  = $3 + $9;
                        if ( io >= mio ) {
                           mio   = io;
                           mseen = $0;
                        }
                     }
                     END {
                        print header;
                        print mseen;
                     }'

               # Find out which device was the busiest.
               mdev="$($PR_diskstats --group-by disk "${BASEDIR}/${PREFIX}-diskstats" \
                  | awk '
                     /[0-9]/ {
                        io  = $3 + $9;
                        if ( io >= mio ) {
                           mio   = io;
                           mdev  = $2;
                        }
                     }
                     END {
                        print mdev;
                     }')"

               # Print the busy% for that device, rounded to the nearest N%, with
               # "." as a marker for a repeated value.
               $PR_diskstats --group-by sample "${BASEDIR}/${PREFIX}-diskstats" \
                  | awk "
                     BEGIN {
                        fuzz = 5;
                        printf \" ${mdev} \"
                     }
                     \$1 = \"${mdev}\" {
                        busy_rounded = fuzz * sprintf(\"%d\", substr(\$15, 1, length(\$15) - 1) / fuzz);
                        if ( printed == 1 && prev == busy_rounded ) {
                           printf \" .\";
                        }
                        else {
                           printf \" %d%%\", busy_rounded;
                           prev    = busy_rounded;
                           printed = 1;
                        }
                     }"
               echo
            else
               echo "    No diskstats file exists"
            fi

            echo "--vmstat--"
            if [ -f "${BASEDIR}/${PREFIX}-vmstat" ]; then
               tail -n 3 "${BASEDIR}/${PREFIX}-vmstat-overall" | $PR_align

               # Figure out which column is 'wa' and print this, similar to the
               # busy% for disks above.
               wa_col="$(awk '/swpd/{for(i=1;i<=NF;++i){if($i=="wa"){print i; exit}}}' "${BASEDIR}/${PREFIX}-vmstat")"
               awk "
                  BEGIN {
                     fuzz = 5;
                     printf \"wa\"
                  }
                  /[0-9]/ {
                     wa_rounded = fuzz * sprintf(\"%d\", \$${wa_col} / fuzz);
                     if ( printed == 1 && prev == wa_rounded ) {
                        printf \" .\";
                     }
                     else {
                        printf \" %d%%\", wa_rounded;
                        prev    = wa_rounded;
                        printed = 1;
                     }
                  }" "${BASEDIR}/${PREFIX}-vmstat"
               echo
            else
               echo "    No vmstat file exists"
            fi

            echo "--innodb--"
            awk '
               /queries inside/ {
                  inside = $0;
               }
               /Main thread/ {
                  main_state = substr($0, index($0, ":") + 2);
               }
               /Pending normal/ {
                  pending_reads += substr($5, 1, length($5) - 1);
                  pending_reads += substr($NF, 1, length($NF) - 1);
               }
               /ibuf aio reads/ {
                  pending_reads += substr($4, 1, length($4) - 1);
                  pending_reads += substr($7, 1, length($7) - 1);
                  pending_reads += $NF;
               }
               /Pending flushes/ {
                  pending_flushes = substr($5, 1, length($5) - 1) + $NF;
               }
               /pending preads/ {
                  pending_reads += $1;
                  pending_writes += $4;
               }
               /pending log writes/ {
                  pending_writes += $1 + $5;
               }
               /Pending reads/ {
                  pending_reads += $NF;
               }
               /Pending writes/ {
                  pending_writes += substr($4, 1, length($4) - 1);
                  pending_writes += substr($7, 1, length($7) - 1);
                  pending_writes += $NF;
               }
               /Log sequence number/ {
                  if ( $NF == 5 ) {
                     lsn = ($4 * (2^32)) + $5;
                  }
                  else {
                     lsn = $4;
                  }
               }
               /Last checkpoint at/ {
                  if ( $NF == 5 ) {
                     chkp = ($4 * (2^32)) + $5;
                  }
                  else {
                     chkp = $4;
                  }
               }
               /END OF INNODB/ {
                  complete = 1;
               }
               /^TRANSACTIONS$/ {
                  tseen = 1;
               }
               /^---TRANSACTION/ {
                  if ( tseen == 1 ) {
                     if ( $2 ~ /,/ ) {
                        status = $3;
                        time   = $4;
                     }
                     else {
                        status = $4;
                        time   = $5;
                     }
                     txns[status]++;
                     if ( time > txntime[status] ) {
                        txntime[status] = time;
                     }
                  }
               }
               /LOCK WAIT/ {
                  if ( tseen == 1 ) {
                     txns["LOCK WAIT"]++;
                     if ( $3 > txntime["LOCK WAIT"] ) {
                        txntime["LOCK WAIT"] = $3;
                     }
                  }
               }
               END {
                  if ( complete != 1 ) {
                     print "    (innodb status is incomplete)";
                  }
                  printf "    txns:";
                  for ( i in txns ) {
                     printf " %dx%s (%ds)", txns[i], i, txntime[i];
                  }
                  print "";
                  if ( inside ) {
                     print "    " inside;
                  }
                  printf "    Main thread: %s, pending reads %d, writes %d, flush %d\n", main_state, pending_reads, pending_writes, pending_flushes;
                  printf "    Log: lsn = %d, chkp = %d, chkp age = %d\n", lsn, chkp, lsn - chkp;
               }
            ' "${BASEDIR}/${PREFIX}-innodbstatus1"
            echo "    Threads are waiting at:"
            awk '/has waited at/ { print $6, $7, $8 }' \
              "${BASEDIR}/${PREFIX}-innodbstatus1" | sort | uniq -c | sort -rn
            echo "    Threads are waiting on:"
            awk '/^[XS]-lock on.*latch/ { print }' \
              "${BASEDIR}/${PREFIX}-innodbstatus1" | sort | uniq -c | sort -rn

            # This section checks for processlist or processlist1 for backwards
            # compatibility with the obsolete pt-collect tool.
            echo "--processlist--"
            local PROCESSLIST_FILE="${BASEDIR}/${PREFIX}-processlist"
            if [ -e "${BASEDIR}/${PREFIX}-processlist1" ]; then
               PROCESSLIST_FILE="${BASEDIR}/${PREFIX}-processlist1"
            fi
            for word in State Command; do
               echo "    $word"
               awk -F: -v column="$word" '
               BEGIN {
                  regex = "^ *" column
               }
               {
                  if ( $1 ~ regex ) {
                     print $2;
                  }
                  # Newer versions of pt-stalk gather several samples. We will
                  # analyze only the first sample.
                  if ( $0 ~ /^TS/ ) {
                     ts++;
                     if (ts > 1) {
                        exit
                     }
                  }
               }' "${PROCESSLIST_FILE}" \
                  | sort | uniq -c | sort -rn | head -n 5
            done

            echo "--stack traces--"
            if [ -e "${BASEDIR}/${PREFIX}-stacktrace" ]; then
               $PR_pmp -l 5 "${BASEDIR}/${PREFIX}-stacktrace" | head -n 5
            else
               echo "    No stack trace file exists"
            fi

            echo "--oprofile--"
            if [ ! -e "${BASEDIR}/${PREFIX}-opreport" ]; then
               echo "    No opreport file exists"
            fi
            test -e "${BASEDIR}/${PREFIX}-opreport" && awk '
               {
                  if ( $1 == "samples" ) {
                     go = 1;
                  }
                  if ( go == 1 ) {
                     print "    " $0;
                     if ( printed++ == 6 ) {
                        exit;
                     }
                  }
               } ' "${BASEDIR}/${PREFIX}-opreport"
            ;;

         LIST)
            ls -lh ${BASEDIR}/${PREFIX}-*
            ;;

         VIEW)
            echo "Viewing all files"
            less -i ${BASEDIR}/${PREFIX}-*
            echo "Press a key to continue or choose a different action"
            ;;

         DISKSTATS)
            echo "Starting $PR_diskstats"
            $PR_diskstats "${BASEDIR}/${PREFIX}-diskstats"
            echo "Press a key to continue or choose a different action"
            ;;

         INNODB)
            echo "Viewing InnoDB files"
            less -i "${BASEDIR}/${PREFIX}-innodbstatus1"
            echo "Press a key to continue or choose a different action"
            ;;

         MEXT)
            echo "Displaying the first 4 samples of SHOW STATUS counters"
            # Grab the first 4 samples by looking for blank lines.
            # I'll rewrite pt-mext and this will be simpler in future.
            # TODO: upgrade, if pt-mext is fixed :)
            awk '/---/{if(i++>12){exit}}{print}' "${BASEDIR}/${PREFIX}-mysqladmin" | $PR_mext -r -- cat - | less -S
            echo "Press a key to continue or choose a different action"
            ;;

         NETWORK)
            echo "Source of connections to port 3306"
            awk '
               /:3306/ {
                  print substr($5, 0, index($5, ":") - 1);
               }
               /TS/ {
                  if ( i++ > 1 ) {
                     # Stop after the first sample
                     exit;
                  }
               }' "${BASEDIR}/${PREFIX}-netstat" | sort | uniq -c | sort -rn
            echo "Status of connections to port 3306"
            awk '
               /:3306/ {
                  print $6;
               }
               /TS/ {
                  if ( i++ > 1 ) {
                     # Stop after the first sample
                     exit;
                  }
               }' "${BASEDIR}/${PREFIX}-netstat" | sort | uniq -c | sort -rn
            echo "Press a key to continue or choose a different action"
            ;;

         INVALID)
            ;;

      esac

      # Capture and handle the interactive key-strokes.
      tput sgr0
      KEY=""
      if ! read -n 1 -s KEY 2>/dev/null; then
         echo "Error while trying to read interactive keystroke command. Exiting."
         exit
      fi
      case "${KEY:-}" in
         j|k)
            PREFIX="$(awk "
               BEGIN {
                  printed = 0;
               }
               {
                  prev=curr;
                  curr=\$1;
                  if ( \"j\" == \"${KEY}\" && prev == \"${PREFIX}\" && curr ~ /./ ) {
                     print curr;
                     printed = 1;
                     exit;
                  }
                  if ( \"k\" == \"${KEY}\" && curr == \"${PREFIX}\" && prev ~ /./ ) {
                     print prev;
                     printed = 1;
                     exit;
                  }
               }
               END {
                  if ( printed == 0 ) {
                     print \"${PREFIX}\";
                  }
               }" $PT_TMPDIR/pt-sift.prefixes)"
            ;;
         1)
            ACTION="DEFAULT"
            ;;
         0)
            ACTION="LIST"
            ;;
         '*')
            ACTION="VIEW"
            ;;
         d)
            ACTION="DISKSTATS"
            ;;
         i)
            ACTION="INNODB"
            ;;
         m)
            ACTION="MEXT"
            ;;
         n)
            ACTION="NETWORK"
            ;;
         q)
            ;;
         '?')
            print_help
            echo "Press any key to continue"
            read -n 1 -s
            ;;
         *)
            echo "Unknown key '${KEY}'; press ? for help"
            ACTION="INVALID"
            ;;
      esac
   done
}

# Execute the program if it was not included from another file.  This makes it
# possible to include without executing, and thus test.
if    [ "${0##*/}" = "$TOOL" ] \
   || [ "${0##*/}" = "bash" -a "${_:-""}" = "$0" ]; then

   mk_tmpdir

   parse_options "$0" "${@:-""}"
   if [ -z "$OPT_HELP" -a -z "$OPT_VERSION" ]; then
      if [ $# -gt 1 ]; then
         option_error "Specify only one PREFIX or DIR"
      fi
   fi
   usage_or_errors "$0"
   po_status=$?
   if [ $po_status -ne 0 ]; then
      [ $OPT_ERRS -gt 0 ] && exit 1
      exit 0
   fi

   main "${@:-""}"

   rm_tmpdir
fi

# ############################################################################
# Documentation
# ############################################################################
:<<'DOCUMENTATION'
=pod

=head1 NAME

pt-sift - Browses files created by pt-stalk.

=head1 SYNOPSIS

Usage: pt-sift FILE|PREFIX|DIRECTORY

pt-sift browses files created by L<pt-stalk>.  If no options are given,
the tool browses all pt-stalk files in C</var/lib/pt-stalk> if that directory
exists, else the current working directory is used.  If a FILE is given,
the tool browses files with the same prefix in the given file's directory.
If a PREFIX is given, the tool browses files in C</var/lib/pt-stalk>
(or the current working directory) with the same prefix.  If a DIRECTORY
is given, the tool browses all pt-stalk files in it.

=head1 RISKS

Percona Toolkit is mature, proven in the real world, and well tested,
but all database tools can pose a risk to the system and the database
server.  Before using this tool, please:

=over

=item * Read the tool's documentation

=item * Review the tool's known L<"BUGS">

=item * Test the tool on a non-production server

=item * Backup your production server and verify the backups

=back

=head1 DESCRIPTION

pt-sift downloads other tools that it might need, such as L<pt-diskstats>,
and then makes a list of the unique timestamp prefixes of all the files in
the directory, as written by the L<pt-stalk> tool.  If the user specified
a timestamp on the command line, then it begins with that sample of data;
otherwise it begins by showing a list of the timestamps and prompting for
a selection.  Thereafter, it displays a summary of the selected sample, and
the user can navigate and inspect with keystrokes.  The keystroke commands
you can use are as follows:

=over

=item * d

Sets the action to start the L<pt-diskstats> tool on the sample's disk
performance statistics.

=item * i

Sets the action to view the first INNODB STATUS sample in less.

=item * m

Displays the first 4 samples of SHOW STATUS counters side by side with the
L<pt-mext> tool.

=item * n

Summarizes the first sample of netstat data in two ways: by originating host,
and by connection state.

=item * j

Select the next timestamp as the active sample.

=item * k

Select the previous timestamp as the active sample.

=item * q

Quit the program.

=item * 1

Sets the action for each sample to the default, which is to view a summary
of the sample.

=item * 0

Sets the action to just list the files in the sample.

=item * *

Sets the action to view all of the sample's files in the less program.

=back

=head1 OPTIONS

=over

=item --help

Show help and exit.

=item --version

Show version and exit.

=back

=head1 ENVIRONMENT

This tool does not use any environment variables.

=head1 SYSTEM REQUIREMENTS

This tool requires Bash v3 and the following programs: pt-diskstats, pt-pmp,
pt-mext, and pt-align.  If these programs are not in your PATH,
they will be fetched from the Internet if curl is available.

=head1 BUGS

For a list of known bugs, see L<http://www.percona.com/bugs/pt-sift>.

Please report bugs at L<https://bugs.launchpad.net/percona-toolkit>.
Include the following information in your bug report:

=over

=item * Complete command-line used to run the tool

=item * Tool L<"--version">

=item * MySQL version of all servers involved

=item * Output from the tool including STDERR

=item * Input files (log/dump/config files, etc.)

=back

If possible, include debugging output by running the tool with C<PTDEBUG>;
see L<"ENVIRONMENT">.

=head1 DOWNLOADING

Visit L<http://www.percona.com/software/percona-toolkit/> to download the
latest release of Percona Toolkit.  Or, get the latest release from the
command line:

   wget percona.com/get/percona-toolkit.tar.gz

   wget percona.com/get/percona-toolkit.rpm

   wget percona.com/get/percona-toolkit.deb

You can also get individual tools from the latest release:

   wget percona.com/get/TOOL

Replace C<TOOL> with the name of any tool.

=head1 AUTHORS

Baron Schwartz

=head1 ABOUT PERCONA TOOLKIT

This tool is part of Percona Toolkit, a collection of advanced command-line
tools for MySQL developed by Percona.  Percona Toolkit was forked from two
projects in June, 2011: Maatkit and Aspersa.  Those projects were created by
Baron Schwartz and primarily developed by him and Daniel Nichter.  Visit
L<http://www.percona.com/software/> to learn about other free, open-source
software from Percona.

=head1 COPYRIGHT, LICENSE, AND WARRANTY

This program is copyright 2011-2015 Percona LLC and/or its affiliates,
2010-2011 Baron Schwartz.

THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.

This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation, version 2; OR the Perl Artistic License.  On UNIX and similar
systems, you can issue `man perlgpl' or `man perlartistic' to read these
licenses.

You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA  02111-1307  USA.

=head1 VERSION

pt-sift 2.2.16

=cut

DOCUMENTATION