openvas-manager 6.0.9-2 / usr / sbin / openvas-certdata-sync

#!/bin/sh
#
# OpenVAS
# $Id: $
# Description: Synchronize with CERT data feed.
#
# Authors:
# Timo Pollmeier <timo.pollmeier@greenbone.net>
#
# Copyright:
# Copyright (C) 2013 Greenbone Networks GmbH
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

POSTGRES=0
SCHEMA_CERT=""
SCHEMA_SCAP=""
RECURSIVE_TRIGGERS_OFF="PRAGMA recursive_triggers = OFF;"
ATTACH_SCAP=""
if [ "SQLITE3" = "POSTGRESQL" ]; then
  POSTGRES=1
  SCHEMA_CERT="cert."
  SCHEMA_SCAP="scap."
  RECURSIVE_TRIGGERS_OFF=""
fi

# configure CERT_DIR where we will sync CERT data
if [ -z "$CERT_DIR" ]; then
  CERT_DIR=/var/lib/openvas/cert-data
fi
CERT_DB="$CERT_DIR/cert.db"

# configure SCAP_DIR where we will get CVE data
if [ -z "$SCAP_DIR" ]; then
  OPENVASSD=`which openvassd`
  if [ -z "$OPENVASSD" ] ; then
    echo "[e] Error: openvassd is not in the path, could not determine SCAP directory."
    exit 1
  else
    # get the parent directory of the plugins
    SCAP_DIR=`openvassd -s | awk -F" = " '/^plugins_folder/ { print $2 }' | sed -s 's/\(^.*\)\/plugins/\1/'`
    # suffix it with "scap-data" which is our target (destination) directory
    SCAP_DIR="$SCAP_DIR/scap-data"
  fi
fi
SCAP_DB="$SCAP_DIR/scap.db"

if [ ! $POSTGRES -eq 1 ] ; then
  ATTACH_SCAP="ATTACH DATABASE '$SCAP_DB' AS scap;"
fi

# private directory
if [ -z "$PRIVATE_SUBDIR" ]
then
  PRIVATE_SUBDIR="private"
fi

# delete options for rsync
RSYNC_DELETE="--delete --exclude \"/cert.db\" --exclude \"$PRIVATE_SUBDIR/\""

# Script and feed information which will be made available to user through
# command line options and automated tools.
SCRIPT_NAME="openvas-certdata-sync"
VERSION=6.0.9
CERT_RES_DIR=/usr/share/openvas/cert
RESTRICTED=0

# Maximum number of retries if database is locked
if [ -z "$MAX_SQL_RETRIES" ]; then
  MAX_SQL_RETRIES="1" # 0 : try only once
fi

# Delay between retries
if [ -z "$SQL_RETRY_DELAY" ]; then
  SQL_RETRY_DELAY="10m" # allowed unit suffixes: see sleep command
fi

TIMESTAMP="$CERT_DIR/timestamp"

if [ -z "$FEED_NAME" ] ; then
  FEED_NAME="OpenVAS CERT Feed"
fi

if [ -z "$FEED_VENDOR" ] ; then
  FEED_VENDOR="The OpenVAS Project"
fi

if [ -z "$FEED_HOME" ] ; then
  FEED_HOME="http://www.openvas.org/"
fi

if [ -z "$OV_CERT_RSYNC_FEED" ]; then
  OV_CERT_RSYNC_FEED=rsync://feed.openvas.org:/cert-data
  # An alternative syntax which might work if the above doesn't:
  # OV_RSYNC_FEED=rsync@feed.openvas.org::cert-data
fi

if [ -z "$TMPDIR" ]; then
  SYNC_TMP_DIR=/tmp
  # If we have mktemp, create a temporary dir (safer)
  if [ -n "`which mktemp`" ]; then
    SYNC_TMP_DIR=`mktemp -t -d openvas-cert-data-sync.XXXXXXXXXX` || { echo "ERROR: Cannot create temporary directory for file download" >&2; exit 1 ; }
    trap "rm -rf $SYNC_TMP_DIR" EXIT HUP INT TRAP TERM
  fi
else
  SYNC_TMP_DIR="$TMPDIR"
fi

do_help () {
  echo "$0: Sync CERT advisory data"
  echo "OpenVAS administrator functions:"
  echo " --migrate      migrate database without downloading feed data"
  echo " --refresh      refresh database without downloading feed data"
  echo " --selftest     perform self-test"
  echo " --identify     display information"
  echo " --version      display version"
  echo " --describe     display current CERT feed info"
  echo " --feedversion   display current CERT feed version"
  echo "Options:"
  if [ $POSTGRES -eq 1 ];
  then
    echo " --database       database name"
  fi
  echo ""
  echo "Environment variables:"
  echo "CERT_DIR             where to place CERT advisories"
  echo "OV_CERT_RSYNC_FEED   URL of rsync feed"
  echo "TMPDIR               temporary directory used to download the files"
  echo "PRIVATE_SUBDIR       subdirectory to exclude from deletion by rsync"
  echo ""
  exit 0
}



CMD_RSYNC=`which rsync`
CMD_SQLITE=`which sqlite3`
SQLITE="sqlite3 -noheader -bail"
TMP_CERT="$SYNC_TMP_DIR/openvas-feed-`date +%F`-$$.tar.bz2"

chk_system_tools () {
  echo "[i] Searching for required system tools (look for warnings/errors)..."

  if [ -z "$CMD_RSYNC" ]; then
    echo "[w] Warning: RSYNC not found";
  fi

  if [ -z "$CMD_SQLITE" ]; then
    echo "[e] Error: sqlite3 not found (required)";
    exit 1
  fi

  if [ -z "$CMD_RSYNC" ]; then
    SELFTEST_FAIL=1
  fi

  echo "[i] If you did not get any warnings, that means you have all tools required"
}

do_rsync () {
  if [ -z "$CMD_RSYNC" ]; then
    echo "[w] rsync not found!"
  else
    echo "[i] Using rsync: $CMD_RSYNC"
    echo "[i] Configured CERT data rsync feed: $OV_CERT_RSYNC_FEED"
    mkdir -p "$CERT_DIR"
    eval "$CMD_RSYNC -ltvrP $RSYNC_DELETE \"$OV_CERT_RSYNC_FEED\" \"$CERT_DIR\""
    if [ $? -ne 0 ] ; then
      echo "Error: rsync failed. Your CERT data might be broken now."
      exit 1
    fi
  fi
}

do_self_test () {
  chk_system_tools
}

do_describe () {
  echo "This script synchronizes a CERT advisory collection with the '$FEED_NAME'."
  echo "The '$FEED_NAME' is provided by '$FEED_VENDOR'."
  echo "Online information about this feed: '$FEED_HOME'."
}

do_feedversion () {
  if [ -r $TIMESTAMP ] ; then
      echo `cat $TIMESTAMP`
  fi
}

show_intro () {
  echo "[i] This script synchronizes a CERT advisory directory with the OpenVAS one."
  if [ $POSTGRES -eq 1 ] ; then
    echo "[i] This script is for the PostgreSQL backend."
  else
    echo "[i] This script is for the SQLite3 backend."
  fi
  echo "[i] CERT dir: $CERT_DIR"
}

do_sync () {
  if [ -z "$CMD_RSYNC" ] ; then
    echo "[w] rsync not found!"
    echo -n "[e] no utility available in PATH environment variable to download CERT data"
    exit 1
  else
    echo "[i] Will use rsync"
    do_rsync
  fi
}

set_interrupt_trap () {
  trap "handle_interrupt X" 2
}

handle_interrupt () {
  echo "$1:X" >&3
}

reset_sql_tries () {
  try_sql=1
  sql_retries=0
}

test_exit_codes () {
  try_sql=0
  if [ -n "$exit_codes" ]
  then
    for item in $exit_codes
    do
      command=`echo "$item" | cut -d':' -f1`
      code=`echo "$item" | cut -d':' -f2`

      if [ "X" = "$code" ]
      then
        echo "[e] $1: Sync script was interrupted"
        exit 1
      elif [ "sqlite3" = "$command" ] \
           && ( [ "5" = "$code" ] || [ "1" = "$code" ] )
      then
        if [ "5" = "$code" ]
        then
          echo "[w] CERT database is locked"
        else
          echo "[w] Could not access CERT database, file may be locked."
        fi

        if [ "$MAX_SQL_RETRIES" -gt "$sql_retries" ]
        then
          sql_retries=$((sql_retries + 1))
          echo "[i] Will try to access database again later in $SQL_RETRY_DELAY. (Retry $sql_retries of $MAX_SQL_RETRIES)"
          try_sql=1
          sleep "$SQL_RETRY_DELAY"
        else
          echo "[e] $1: Gave up trying to access CERT database."
          exit 1
        fi
      else
        echo "[e] $1: $command exited with code $code"
        exit 1
      fi
    done
  fi
}

test_sql_exit () {
  exit_code=$?
  try_sql=0
  if ( [ "5" = "$exit_code" ] || [ "1" = "$exit_code" ] )
  then
    if [ "5" = "$exit_code" ]
    then
      echo "[w] CERT database is locked."
    else
      echo "[w] Could not access CERT database, file may be locked."
    fi

    if [ "$MAX_SQL_RETRIES" -gt "$sql_retries" ]
    then
      sql_retries=$((sql_retries + 1))
      echo "[i] Will try to access database again later in $SQL_RETRY_DELAY. (Retry $sql_retries of $MAX_SQL_RETRIES)"
      try_sql=1
      sleep "$SQL_RETRY_DELAY"
    else
      echo "[e] $1: Gave up trying to access CERT database."
      exit 1
    fi
  elif [ 0 -ne "$exit_code" ]
  then
    echo "[e] $1: sqlite3 exited with code $exit_code."
    exit 1
  fi
}

reinit () {
  echo "[i] Major change in internal CERT data structures."
  echo "[i] Reinitialization of database necessary."
  echo "[i] This update might take a while.."
  reset_sql_tries
  until [ "$try_sql" -eq 0 ]
  do
    sql < $CERT_RES_DIR/cert_db_init.sql
    test_sql_exit "Could not reinitialize CERT database"
  done
}

PSQL=""
psql_setup () {
  PSQL="psql -v ON_ERROR_STOP=1 -q --pset pager=off --no-align -d $1 -t"
}

sql_pg () {
  if [ "$#" -gt 0 ]
  then
    $PSQL -c "$1"
  else
    $PSQL -f -
  fi
  exit_code=$?
  if [ 0 -ne "$exit_code" ]
  then
    echo "[e] psql exited with code $exit_code for sql: $1."
    exit 1
  fi
}

sql_scap () {
  if [ $POSTGRES -eq 1 ]
  then
    if [ "$#" -gt 0 ]
    then
      sql_pg "$1"
    else
      sql_pg $*
    fi
  else
    if [ "$#" -gt 0 ]
    then
      $SQLITE $SCAP_DB "$1"
    else
      $SQLITE $SCAP_DB $*
    fi
  fi
}

sql () {
  if [ $POSTGRES -eq 1 ]
  then
    if [ "$#" -gt 0 ]
    then
      sql_pg "$1"
    else
      sql_pg $*
    fi
  else
    if [ "$#" -gt 0 ]
    then
      $SQLITE $CERT_DB "$1"
    else
      $SQLITE $CERT_DB $*
    fi
  fi
}

check_db_version () {
  DB_VERSION=`sql "select value from ${SCHEMA_CERT}meta where name = 'database_version';" 2>/dev/null | tr -d '\n\r' || echo 0`
  case "$DB_VERSION" in
    0) reinit;;
    1) reinit;;
    2) reinit;;
    3) reinit;;
    4) reinit;;
    5) reinit;;
  esac
}

scap_db_exists () {
  if [ $POSTGRES -eq 1 ] ;
  then
    EXISTS=`sql_pg "SELECT exists (SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'scap');" 2>/tmp/ERR | tr -d '\n\r' || echo 0`
    if [ "$EXISTS" = "f" ] ;
    then
      return 1;
    fi
    return 0;
  elif [ -f $SCAP_DB ]
  then
    return 0;
  fi
  return 1;
}

cert_db_exists () {
  if [ $POSTGRES -eq 1 ] ;
  then
    EXISTS=`sql_pg "SELECT exists (SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'cert');" 2>/tmp/ERR | tr -d '\n\r' || echo 0`
    if [ "$EXISTS" = "f" ] ;
    then
      return 1;
    fi
    return 0;
  elif [ -f $CERT_DB ]
  then
    return 0;
  fi
  return 1;
}

cert_db_remove () {
  if [ $POSTGRES -eq 1 ] ;
  then
    sql_pg "DROP SCHEMA IF EXISTS cert CASCADE;"
  else
    rm -f $CERT_DB
  fi
}

update_cvss () {
  if ! scap_db_exists;
  then
    echo "[w] SCAP database not found. Cannot update max_cvss."
  else
    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      SCAP_DB_LASTUPDATE=`sql_scap "SELECT value FROM ${SCHEMA_SCAP}meta WHERE name = 'last_update';"`
      test_sql_exit "Could not get last_update timestamp from SCAP database"
    done

    if [ 0 -ne "$updated_cb" ] || [ $SCAP_DB_LASTUPDATE -gt $DB_LASTUPDATE ]
    then
      echo "[i] Updating Max CVSS for CERT-Bund"
      reset_sql_tries
      until [ "$try_sql" -eq 0 ]
      do
        sql "${ATTACH_SCAP}
             ${RECURSIVE_TRIGGERS_OFF}
             UPDATE ${SCHEMA_CERT}cert_bund_advs
               SET max_cvss =
                   (
                     SELECT max(cvss)
                     FROM scap.cves
                     WHERE name IN
                       (
                         SELECT cve_name
                         FROM ${SCHEMA_CERT}cert_bund_cves
                         WHERE adv_id=cert_bund_advs.id
                       )
                       AND cvss != 0.0
                   );"
        test_sql_exit "Could not update CVSS scores for CERT-Bund Advisories"
      done
    else
      echo "[i] No CERT-Bund advisories updated and CERT DB newer than SCAP DB. Skipping CVSS recalculation."
    fi

    if [ 0 -ne "$updated_dfn" ] || [ $SCAP_DB_LASTUPDATE -gt $DB_LASTUPDATE ]
    then
      echo "[i] Updating Max CVSS for DFN-CERT"
      reset_sql_tries
      until [ "$try_sql" -eq 0 ]
      do
        sql "${ATTACH_SCAP}
             ${RECURSIVE_TRIGGERS_OFF}
             UPDATE ${SCHEMA_CERT}dfn_cert_advs
               SET max_cvss =
                   (
                     SELECT max(cvss)
                     FROM scap.cves
                     WHERE name IN
                       (
                         SELECT cve_name
                         FROM ${SCHEMA_CERT}dfn_cert_cves
                         WHERE adv_id=dfn_cert_advs.id
                       )
                       AND cvss != 0.0
                   );"
        test_sql_exit "Could not update CVSS scores for DFN-CERT Advisories"
      done
    else
      echo "[i] No DFN-CERT advisories updated and CERT DB newer than SCAP DB. Skipping CVSS recalculation."
    fi
  fi
}

update_sec_db () {
  if [ -z "$updated_cb" ]
  then
    updated_cb=0
  fi

  if [ -z "$updated_dfn" ]
  then
    updated_dfn=0
  fi

  if cert_db_exists
  then
    check_db_version

    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      CB_REFDATE=`sql "SELECT max(modification_time) from ${SCHEMA_CERT}cert_bund_advs;"`
      test_sql_exit "Could not get CERT-Bund reference date from database"
    done

    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      DFN_REFDATE=`sql "SELECT max(modification_time) from ${SCHEMA_CERT}dfn_cert_advs;"`
      test_sql_exit "Could not get DFN-CERT reference date from database"
    done
  else
    echo "[i] Initializing CERT advisory database"
    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      sql < $CERT_RES_DIR/cert_db_init.sql
      test_sql_exit "Could not initialize CERT database"
    done
    DB_LASTUPDATE=0
  fi

  if [ -z "$CB_REFDATE" ]
  then
    CB_REFDATE=0
  fi

  if [ -z "$DFN_REFDATE" ]
  then
    DFN_REFDATE=0
  fi

  reset_sql_tries
  until [ "$try_sql" -eq 0 ]
  do
    DB_LASTUPDATE=`sql "SELECT value FROM ${SCHEMA_CERT}meta WHERE name = 'last_update';"`
    test_sql_exit "Could not get last_update timestamp from CERT database"
  done

  if [ -z "$DB_LASTUPDATE" ]
  then
    # Happens when initial sync was aborted
    echo "Error: Inconsistent data. Resetting CERT database."
    cert_db_remove

    reset_sql_tries
    until [ "$try_sql" -eq 0 ]
    do
      sql < $CERT_RES_DIR/cert_db_init.sql
      test_sql_exit "Could not reinitialize CERT database"
    done

    CB_REFDATE=0
    DFN_REFDATE=0
    DB_LASTUPDATE=0
  fi

  # Update CERT-Bund
  xmlcount=$(ls $CERT_DIR/CB-K*.xml 2> /dev/null | wc -l)
  if [ $xmlcount -ne 0 ]
  then
    for certfile in `ls $CERT_DIR/CB-K*.xml`
    do
      filedate=`stat -c "%Y" $certfile | cut -d " " -f 1 | tr -d "-"`
      filedate=$(( $filedate - ( $filedate % 60 ) ))
      if [ $filedate -gt $DB_LASTUPDATE ]
      then
        echo "[i] Updating $certfile"

        reset_sql_tries
        until [ "$try_sql" -eq 0 ]
        do
          exec 4>&1
            exit_codes=$(
              (
                set_interrupt_trap
                (xsltproc --stringparam refdate "$CB_REFDATE" $CERT_RES_DIR/cert_bund_update.xsl $certfile || echo "xsltproc:$?" >&3) | \
                (sql | sed '/^\s*$/d' || echo "sqlite3:$?" >&3)
              ) 3>&1 >&4
            )
          exec 4>&-
          test_exit_codes "Update of CERT-Bund Advisories failed at file '$certfile'"
        done
        updated_cb=1
      else
        echo "[i] Skipping $certfile, file is older than last revision"
      fi
    done
  else
    echo "[w] No CERT-Bund advisories found in $CERT_DIR"
  fi

  # Update DFN-CERT
  xmlcount=$(ls $CERT_DIR/dfn-cert-*.xml 2> /dev/null | wc -l)
  if [ $xmlcount -ne 0 ]
  then
    for certfile in `ls $CERT_DIR/dfn-cert-*.xml`
    do
      filedate=`stat -c "%Y" $certfile | cut -d " " -f 1 | tr -d "-"`
      filedate=$(( $filedate - ( $filedate % 60 ) ))
      if [ $filedate -gt $DB_LASTUPDATE ]
      then
        echo "[i] Updating $certfile"

        reset_sql_tries
        until [ "$try_sql" -eq 0 ]
        do
          exec 4>&1
            exit_codes=$(
              (
                set_interrupt_trap
                (xsltproc --stringparam refdate "$DFN_REFDATE" $CERT_RES_DIR/dfn_cert_update.xsl $certfile || echo "xsltproc:$?" >&3) | \
                (sql | sed '/^\s*$/d' || echo "sqlite3:$?" >&3)
              ) 3>&1 >&4
            )
          exec 4>&-
          test_exit_codes "Update of DFN-CERT Advisories failed at file '$certfile'"
        done
        updated_dfn=1
      else
        echo "[i] Skipping $certfile, file is older than last revision"
      fi
    done
  else
    echo "[w] No DFN-CERT advisories found in $CERT_DIR"
  fi

  update_cvss

  LAST_UPDATE_TIMESTAMP=`sed 's/^\(.\{8\}\)/\1 /' $TIMESTAMP | env TZ="UTC" date +%s -f -`
  reset_sql_tries
  until [ "$try_sql" -eq 0 ]
  do
    sql "UPDATE ${SCHEMA_CERT}meta SET value ='$LAST_UPDATE_TIMESTAMP' WHERE name = 'last_update';"
    test_sql_exit "Could not update last_update timestamp in CERT database"
  done
}

do_refresh () {
  echo "[i] Refreshing database without feed sync."
  update_sec_db
}

psql_setup "tasks"
if [ -n "$1" ]; then
  while test $# -gt 0; do
    case "$1" in
      --help)
        do_help
        exit 0
        ;;
      --refresh)
        do_refresh
        exit 0
        ;;
      --migrate)
        do_refresh
        exit 0
        ;;
      --check)
        exit 0
        ;;
      --version)
        echo $VERSION
        exit 0
        ;;
      --identify)
        echo "CERTSYNC|$SCRIPT_NAME|$VERSION|$FEED_NAME|$RESTRICTED|CERTSYNC"
        exit 0
        ;;
      --selftest)
        SELFTEST_FAIL=0
        do_self_test
        exit $SELFTEST_FAIL
        ;;
      --describe)
        do_describe
        exit 0
        ;;
      --feedversion)
        do_feedversion
        exit 0
        ;;
      --database)
		if [ $POSTGRES -eq 1 ] ;
		then
		  shift
          if [ $# -gt 0 ]; then
		    psql_setup $1
	      else
			echo "[e] --database argument missing"
			exit 1
		  fi
		else
		  echo "[e] --database option available only for Postgres"
		  exit 1
		fi
        ;;
    esac
    shift
  done
fi

show_intro
do_sync
update_sec_db

exit 0