python3-openstack-doc-tools 0.34.0-2 / usr / lib / python3 / dist-packages / os_doc_tools / doctest.py

#!/usr/bin/env python

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

'''
Usage:
    test.py [path]

Validates all xml files against the DocBook 5 RELAX NG schema, and
attempts to build all books.

Options:
    path     Root directory, defaults to <repo root>/doc

Ignores pom.xml files and subdirectories named "target".

Requires:
    - Python 2.7 or greater
    - lxml Python library
    - Maven
'''

import gzip
import multiprocessing
import operator
import os
import re
import shutil
import six
import subprocess
import sys
import time

if six.PY3:
    from urllib import request as urllib2
else:
    import urllib2

from lxml import etree
from oslo_config import cfg

import os_doc_tools
from os_doc_tools import index
from os_doc_tools import jsoncheck
from os_doc_tools.openstack.common import log


LOG = log.getLogger('openstack-doc-test')

# These are files that are known to not pass syntax or niceness checks
# Add values via --file-exceptions.
FILE_EXCEPTIONS = []

# These are files that are known to not be in DocBook XML format.
# Add values via --build-file-exceptions.
BUILD_FILE_EXCEPTIONS = []

# These are books that we aren't checking yet.
BOOK_EXCEPTIONS = []

# These URLs will not be checked for reachability. Add values via
# --url-exception.
URL_EXCEPTIONS = []

# Mappings from books to build directories under target
BOOK_MAPPINGS = {}

# Mappings from books to publish directories
BOOK_PUBLISH_MAPPINGS = {}

RESULTS_OF_BUILDS = []

# List of recognized (allowable) os profiling directives.
KNOWN_OS_VALUES = ["debian",
                   "centos",
                   "fedora",
                   "opensuse",
                   "rhel",
                   "sles",
                   "ubuntu"]


# List of recognized (allowable) audience profiling directives.
KNOWN_AUDIENCE_VALUES = ["enduser",
                         "adminuser",
                         "installer",
                         "webpage"]

OS_DOC_TOOLS_DIR = os.path.dirname(__file__)
# NOTE(jaegerandi): BASE_RNG needs to end with '/', otherwise
# the etree.parse call in get_wadl_schema will fail.
BASE_RNG = os.path.join(OS_DOC_TOOLS_DIR, 'resources/')
RACKBOOK_RNG = os.path.join(BASE_RNG, 'rackbook.rng')
DOCBOOKXI_RNG = os.path.join(BASE_RNG, 'docbookxi.rng')
WADL_RNG = os.path.join(BASE_RNG, 'wadl.rng')
WADL_XSD = os.path.join(BASE_RNG, 'wadl.xsd')

SCRIPTS_DIR = os.path.join(OS_DOC_TOOLS_DIR, 'scripts')


def get_schema(is_api_site=False):
    """Return the DocBook RELAX NG schema."""
    if is_api_site:
        url = RACKBOOK_RNG
    else:
        url = DOCBOOKXI_RNG
    relaxng_doc = etree.parse(url)
    return etree.RelaxNG(relaxng_doc)


def get_wadl_schema():
    """Return the Wadl schema."""
    # NOTE(jaegerandi): We could use the RelaxNG instead
    # like follows but this gives quite some errors at the
    # moment, so only validate using the XMLSchema
    # url = WADL_RNG
    # relaxng_doc = etree.parse(url, base_url=BASE_RNG)
    # return etree.RelaxNG(relaxng_doc)
    url = WADL_XSD
    schema = etree.parse(url, base_url=BASE_RNG)
    return etree.XMLSchema(schema)


def validation_failed(schema, doc):
    """Return True if the parsed doc fails against the schema.

    This will ignore validation failures of the type: IDREF attribute linkend
    references an unknown ID. This is because we are validating individual
    files that are being imported, and sometimes the reference isn't present
    in the current file.
    """
    return (not schema.validate(doc) and
            any(log.type_name != "DTD_UNKNOWN_ID" for log in schema.error_log))


def verify_section_tags_have_xmlid(doc):
    """Check that all section tags have an xml:id attribute.

    Will throw an exception if there's at least one missing.
    """
    ns = {"docbook": "http://docbook.org/ns/docbook"}
    for node in doc.xpath('//docbook:section', namespaces=ns):
        if "{http://www.w3.org/XML/1998/namespace}id" not in node.attrib:
            raise ValueError("section missing xml:id attribute, line %d" %
                             node.sourceline)


def verify_attribute_profiling(doc, attribute, known_values):
    """Check for conflicts in attribute profiling.

    Check for elements with attribute profiling set that conflicts with
    the attribute profiling of nodes below them in the DOM
    tree. This picks up cases where content is accidentally
    omitted via conflicting profiling. Checks known_values also for
    supported profiling values.
    """

    msg = []
    ns = {"docbook": "http://docbook.org/ns/docbook"}
    path = '//docbook:*[@%s]' % attribute
    for parent in doc.xpath(path, namespaces=ns):
        p_tag = parent.tag
        p_line = parent.sourceline
        p_att_list = parent.attrib[attribute].split(';')

        for att in p_att_list:
            if att not in known_values:
                msg.append(
                    "'%s' is not a recognized %s profile on line %d." %
                    (att, attribute, p_line))

        cpath = './/docbook:*[@%s]' % attribute
        for child in parent.xpath(cpath, namespaces=ns):
            c_tag = child.tag
            c_line = child.sourceline
            c_att_list = child.attrib[attribute].split(';')
            for att in c_att_list:
                if att not in p_att_list:
                    len_ns = len("{http://docbook.org/ns/docbook}")
                    msg.append(
                        "%s %s profiling (%s) conflicts with %s "
                        "profiling of %s on line %d." %
                        (p_tag[len_ns:], attribute, p_att_list,
                         attribute, c_tag[len_ns:], c_line))
    if msg:
        raise ValueError("\n     ".join(msg))


def verify_profiling(doc):
    """"Check profiling information."""
    verify_attribute_profiling(doc, "os", KNOWN_OS_VALUES)
    verify_attribute_profiling(doc, "audience", KNOWN_AUDIENCE_VALUES)


def verify_newline_end_of_file(docfile):
    """Check that there is a newline at the end of file."""

    with open(docfile, 'r') as fp:
        lines = fp.readlines()
        lastline = lines[-1]

    if not lastline.endswith('\n'):
        raise ValueError('last line of a file must end with a \\n')


def verify_unicode_niceness(docfile):
    """Check that no unwanted unicode charaters are used."""

    invalid_characters = '“”‘’―—'

    affected_lines = []
    lc = 0
    for line in open(docfile, 'r'):
        lc += 1
        any(c in line for c in invalid_characters)

    if affected_lines:
        raise ValueError("unwanted unicode charaters (one of %s) "
                         "found in line(s): %" % (" ".join(invalid_characters),
                                                  ", ".join(affected_lines)))


def verify_whitespace_niceness(docfile):
    """Check that no unnecessary whitespaces are used."""
    checks = [
        re.compile(".*\s+\n$"),
    ]

    elements = [
        'listitem',
        'para',
        'td',
        'th',
        'command',
        'literal',
        'title',
        'caption',
        'filename',
        'userinput',
        'programlisting'
    ]

    for element in elements:
        checks.append(re.compile(".*<%s>\s+[\w\-().:!?{}\[\]]+.*\n"
                                 % element))
        checks.append(re.compile(".*[\w\-().:!?{}\[\]]+\s+<\/%s>.*\n"
                                 % element))

    lc = 0
    affected_lines = []
    tab_lines = []
    nbsp_lines = []
    for line in open(docfile, 'r'):
        lc = lc + 1
        if '\t' in line:
            tab_lines.append(str(lc))

        if '\xc2\xa0' in line:
            nbsp_lines.append(str(lc))
        for check in checks:
            if check.match(line) and lc not in affected_lines:
                affected_lines.append(str(lc))

    msg = ""
    if nbsp_lines:
        msg = "non-breaking space found in lines (use &nbsp;): %s" % (
            ", ".join(nbsp_lines))
    if affected_lines:
        if (msg):
            msg += "\n    "
        msg += ("trailing or unnecessary whitespaces found in lines: %s"
                % (", ".join(affected_lines)))
    if tab_lines:
        if (msg):
            msg += "\n    "
        msg += "tabs found in lines: %s" % ", ".join(tab_lines)

    if msg:
        raise ValueError(msg)


def verify_valid_links(doc):
    """Check that all linked URLs are reachable

    Will throw an exception if there's at least one unreachable URL.
    """
    ns = {"docbook": "http://docbook.org/ns/docbook",
          'xlink': 'http://www.w3.org/1999/xlink'}
    msg = []
    for node in doc.xpath('//docbook:link', namespaces=ns):
        try:
            url = node.attrib['{http://www.w3.org/1999/xlink}href']
        except Exception:
            continue

        if url in URL_EXCEPTIONS:
            continue

        try:
            urllib2.urlopen(url)
        except urllib2.HTTPError as e:
            # Ignore some error codes:
            # 403 (Forbidden) since it often means that the user-agent
            # is wrong.
            # 503 (Service Temporarily Unavailable)
            if e.code not in [403, 503]:
                e_line = node.sourceline
                msg.append("URL %s not reachable at line %d, error %s" % (
                    url, e_line, e))
        except urllib2.URLError as e:
            e_line = node.sourceline
            msg.append("URL %s invalid at line %d, error %s" % (
                url, e_line, e))

    if msg:
        raise ValueError("\n    ".join(msg))


def error_message(error_log):
    """Return a string that contains the error message.

    We use this to filter out false positives related to IDREF attributes
    """
    errs = [str(x) for x in error_log if x.type_name != 'DTD_UNKNOWN_ID']

    # Reverse output so that earliest failures are reported first
    errs.reverse()
    return "\n".join(errs)


def www_touched():
    """Check whether files in www directory are touched."""

    try:
        git_args = ["git", "diff", "--name-only", "HEAD~1", "HEAD"]
        modified_files = subprocess.check_output(
            git_args,
            universal_newlines=True).strip().split()
    except (subprocess.CalledProcessError, OSError) as e:
        print("git failed: %s" % e)
        sys.exit(1)

    www_changed = False
    other_changed = False
    for f in modified_files:
        if f.startswith("www/"):
            www_changed = True
        else:
            other_changed = True

    return www_changed and not other_changed


def only_po_touched():
    """Check whether only files in locale directory are touched."""

    try:
        git_args = ["git", "diff", "--name-only", "HEAD~1", "HEAD"]
        modified_files = subprocess.check_output(
            git_args,
            universal_newlines=True).strip().split()
    except (subprocess.CalledProcessError, OSError) as e:
        print("git failed: %s" % e)
        sys.exit(1)

    locale_changed = False
    other_changed = False
    for f in modified_files:
        if "/locale/" in f and f.endswith((".po", ".pot")):
            locale_changed = True
        else:
            other_changed = True

    return locale_changed and not other_changed


def only_rst_touched():
    """Check whether only RST files are touched."""

    try:
        git_args = ["git", "diff", "--name-only", "HEAD~1", "HEAD"]
        modified_files = subprocess.check_output(
            git_args,
            universal_newlines=True).strip().split()
    except (subprocess.CalledProcessError, OSError) as e:
        print("git failed: %s" % e)
        sys.exit(1)

    rst_changed = False
    other_changed = False
    for f in modified_files:
        if f.endswith(".rst"):
            rst_changed = True
        else:
            other_changed = True

    return rst_changed and not other_changed


def check_modified_affects_all(rootdir):
    """Check whether special files were modified.

    There are some special files where we should rebuild all books
    if either of these is touched.
    """

    os.chdir(rootdir)

    try:
        git_args = ["git", "diff", "--name-only", "HEAD~1", "HEAD"]
        modified_files = subprocess.check_output(
            git_args,
            universal_newlines=True).strip().split()
    except (subprocess.CalledProcessError, OSError) as e:
        print("git failed: %s" % e)
        sys.exit(1)

    special_files = [
        # Top-Level pom.xml
        "pom.xml",
        # doc/pom.xml in openstack-manuals
        "doc/pom.xml"
    ]

    for f in modified_files:
        if f in special_files or f.endswith('.ent'):
            if cfg.CONF.verbose:
                print("File %s modified, this affects all books." % f)
            return True

    return False


def get_modified_files(rootdir, filtering=None):
    """Get modified files below doc directory."""

    # There are several tree traversals in this program that do a
    # chdir, we need to run this git command always from the rootdir,
    # so assure that.
    os.chdir(rootdir)

    try:
        git_args = ["git", "diff", "--name-only", "--relative", "HEAD~1",
                    "HEAD"]
        if filtering is not None:
            git_args.append(filtering)
        modified_files = subprocess.check_output(
            git_args,
            universal_newlines=True).strip().split()
    except (subprocess.CalledProcessError, OSError) as e:
        print("git failed: %s" % e)
        sys.exit(1)
    return modified_files


def filter_dirs(dirs):
    """Return list of directories to descend into."""

    # Don't descend into 'locale', 'target', and 'publish-docs'
    # subdirectories and filter out any dot directories

    return [d for d in dirs if (d != 'target' and
                                d != 'publish-docs' and
                                d != 'locale' and
                                not d.startswith('.'))]


def check_deleted_files(rootdir, file_exceptions, verbose):
    """Checking that no removed files are referenced."""

    print("Checking that no removed files are referenced...")
    deleted_files = get_modified_files(rootdir, "--diff-filter=D")
    if not deleted_files:
        print("No files were removed.\n")
        return 0

    if verbose:
        print(" Removed files:")
        for f in deleted_files:
            print("   %s" % f)

    deleted_files = [os.path.abspath(x) for x in deleted_files]
    no_checked_files = 0

    # Figure out whether files were included anywhere
    missing_reference = False

    for root, dirs, files in os.walk(rootdir):
        dirs[:] = filter_dirs(dirs)

        os.chdir(root)

        for f in files:
            if not is_testable_xml_file(f, file_exceptions):
                continue

            path = os.path.abspath(os.path.join(root, f))
            try:
                doc = etree.parse(path)
            except etree.XMLSyntaxError as e:
                print(" Warning: file %s is invalid XML: %s" % (path, e))
                continue

            no_checked_files = no_checked_files + 1

            # Check for inclusion of files as part of imagedata
            for node in doc.findall(
                    '//{http://docbook.org/ns/docbook}imagedata'):
                href = node.get('fileref')
                if (f not in file_exceptions and
                        os.path.abspath(href) in deleted_files):
                    print("  File %s has imagedata href for deleted "
                          "file %s" % (f, href))
                    missing_reference = True

                    break

            # Check for inclusion of files as part of xi:include
            ns = {"xi": "http://www.w3.org/2001/XInclude"}
            for node in doc.xpath('//xi:include', namespaces=ns):
                href = node.get('href')
                if (os.path.abspath(href) in deleted_files):
                    print("  File %s has an xi:include on deleted file %s"
                          % (f, href))
                    missing_reference = True

    if missing_reference:
        print("Check failed, %d files were removed, "
              "%d files checked.\n"
              % (len(deleted_files), no_checked_files))
        return 1

    print("Check passed, %d files were removed, "
          "%d files checked.\n"
          % (len(deleted_files), no_checked_files))
    return 0


def indent(text, spaces):
    """Indent each line of a string by given number of spaces.

    The text argument is passed through str() first to turn objects such
    as exceptions into strings.
    """
    return ''.join((spaces * ' ') + i for i in str(text).splitlines(True))


def validate_one_json_file(rootdir, path, verbose, check_syntax,
                           check_niceness):
    """Validate a single JSON file."""

    any_failures = False
    if verbose:
        print(" Validating %s" % os.path.relpath(path, rootdir))

    try:
        if check_syntax:
            jsoncheck.check_syntax(path)
    except ValueError as e:
        any_failures = True
        print("  Invalid JSON file %s:\n%s" %
              (os.path.relpath(path, rootdir), indent(e, 4)))
    else:
        try:
            if check_niceness:
                jsoncheck.check_formatting(path)
        except ValueError as e:
            any_failures = True
            print("  %s:\n%s" % (os.path.relpath(path, rootdir),
                                 indent(e, 4)))

    try:
        if check_niceness:
            verify_whitespace_niceness(path)
            verify_newline_end_of_file(path)
            verify_unicode_niceness(path)
    except ValueError as e:
        any_failures = True
        print("  %s: %s" % (os.path.relpath(path, rootdir), e))

    return any_failures


def validate_one_file(schema, rootdir, path, verbose,
                      check_syntax, check_niceness, check_links,
                      validate_schema):
    """Validate a single file."""
    # We pass schema in as a way of caching it, generating it is expensive

    any_failures = False
    if verbose:
        print(" Validating %s" % os.path.relpath(path, rootdir))
    try:
        if check_syntax or check_links:
            doc = etree.parse(path)
            if check_syntax:
                if validate_schema:
                    if validation_failed(schema, doc):
                        any_failures = True
                        print(error_message(schema.error_log))
                    verify_section_tags_have_xmlid(doc)
                    verify_profiling(doc)
            if check_links:
                    verify_valid_links(doc)
        if check_niceness:
            verify_whitespace_niceness(path)
            verify_newline_end_of_file(path)
            verify_unicode_niceness(path)
    except etree.XMLSyntaxError as e:
        any_failures = True
        print("  %s: %s" % (os.path.relpath(path, rootdir), e))
    except ValueError as e:
        any_failures = True
        print("  %s: %s" % (os.path.relpath(path, rootdir), e))

    return any_failures


def is_testable_xml_file(path, exceptions):
    """Returns true if file ends with .xml and is not a pom.xml file."""

    filename = os.path.basename(path)
    return (filename.endswith('.xml') and not filename == 'pom.xml' and
            filename not in exceptions)


def is_testable_file(path, exceptions):
    """Returns true if file is in some XML format we handle

    Skips pom.xml files as well since those are not handled.
    """

    filename = os.path.basename(path)
    return (filename.endswith(('.xml', '.xsd', '.xsl', '.wadl',
                               '.xjb', '.json')) and
            not filename == 'pom.xml' and filename not in exceptions)


def is_wadl(filename):
    """Returns true if file ends with .wadl."""

    return filename.endswith('.wadl')


def is_json(filename):
    """Returns true if file ends with .json."""

    return filename.endswith('.json')


def validate_individual_files(files_to_check, rootdir, verbose,
                              check_syntax=False, check_niceness=False,
                              check_links=False, is_api_site=False):
    """Validate list of files."""

    schema = get_schema(is_api_site)
    if is_api_site:
        wadl_schema = get_wadl_schema()

    any_failures = False
    no_validated = 0
    no_failed = 0

    checks = []
    if check_links:
        checks.append("valid URL links")
    if check_niceness:
        checks.append("niceness")
    if check_syntax:
        checks.append("syntax")
    print("Checking files for %s..." % (", ".join(checks)))

    for f in files_to_check:
        validate_schema = True
        if is_api_site:
            # Files ending with ".xml" in subdirectories of
            # wadls and samples files are not docbook files.
            if (f.endswith('.xml') and
               ("wadls" in f or "samples" in f)):
                validate_schema = False
            # Right now we can only validate docbook .xml
            # and .wadl files with a schema
            elif not f.endswith(('.wadl', '.xml')):
                validate_schema = False

        if is_json(f):
            any_failures = validate_one_json_file(rootdir, f,
                                                  verbose,
                                                  check_syntax,
                                                  check_niceness)
        elif (is_api_site and is_wadl(f)):
            any_failures = validate_one_file(wadl_schema, rootdir, f,
                                             verbose,
                                             check_syntax,
                                             check_niceness,
                                             check_links,
                                             validate_schema)
        else:
            any_failures = validate_one_file(schema, rootdir, f,
                                             verbose,
                                             check_syntax,
                                             check_niceness,
                                             check_links,
                                             validate_schema)
        if any_failures:
            no_failed = no_failed + 1
        no_validated = no_validated + 1

    if no_failed > 0:
        print("Check failed, validated %d files with %d failures.\n"
              % (no_validated, no_failed))
        return 1
    else:
        print("Check passed, validated %d files.\n" % no_validated)
    return 0


def validate_modified_files(rootdir, exceptions, verbose,
                            check_syntax=False, check_niceness=False,
                            check_links=False, is_api_site=False):
    """Validate list of modified, testable files."""

    # Do not select deleted files, just Added, Copied, Modified, Renamed,
    # or Type changed
    modified_files = get_modified_files(rootdir, "--diff-filter=ACMRT")
    modified_files = [f for f in modified_files if
                      is_testable_file(f, exceptions)]

    return validate_individual_files(modified_files, rootdir,
                                     verbose,
                                     check_syntax, check_niceness,
                                     check_links, is_api_site)


def validate_all_files(rootdir, exceptions, verbose,
                       check_syntax, check_niceness=False,
                       check_links=False, is_api_site=False):
    """Validate all testable files (XML, WADL, JSON, etc.)."""

    files_to_check = []

    for root, dirs, files in os.walk(rootdir):
        dirs[:] = filter_dirs(dirs)

        for f in files:
            # Ignore maven files, which are called pom.xml
            if is_testable_file(f, exceptions):
                path = os.path.abspath(os.path.join(root, f))
                files_to_check.append(path)

    return validate_individual_files(files_to_check, rootdir,
                                     verbose,
                                     check_syntax, check_niceness,
                                     check_links, is_api_site)


def logging_build_book(result):
    """Callback for book building."""
    RESULTS_OF_BUILDS.append(result)


def get_gitroot():
    """Return path to top-level of git repository."""

    try:
        git_args = ["git", "rev-parse", "--show-toplevel"]
        gitroot = subprocess.check_output(git_args,
                                          universal_newlines=True).rstrip()
    except (subprocess.CalledProcessError, OSError) as e:
        print("git failed: %s" % e)
        sys.exit(1)
    return gitroot


def print_gitinfo():
    """Print information about repository and change."""

    try:
        git_cmd = ["git", "rev-parse", "--abbrev-ref", "HEAD"]
        gitbranch = subprocess.check_output(git_cmd,
                                            universal_newlines=True).rstrip()
        git_cmd = ["git", "show", "--format=%s", "-s"]
        gitsubject = subprocess.check_output(git_cmd,
                                             universal_newlines=True).rstrip()
        git_cmd = ["git", "show", "--format=%an", "-s"]
        gitauthor = subprocess.check_output(git_cmd,
                                            universal_newlines=True).rstrip()
    except (subprocess.CalledProcessError, OSError) as e:
        print("git failed: %s" % e)
        sys.exit(1)
    print("Testing patch:")
    print("  Title: %s" % gitsubject)
    print("  Author: %s" % gitauthor)
    print("  Branch: %s" % gitbranch)


def get_publish_path():
    """Return path to use of publishing books."""

    return os.path.join(get_gitroot(), 'publish-docs')


def ignore_for_publishing(_, names):
    """Return list of files that should be ignored for publishing."""

    # Ignore:
    # - all files ending with .xml with the exception of atom.xml
    # - all files ending with .fo
    # The directory named wadls

    f = [n for n in names if ((n.endswith('.xml') and n != 'atom.xml')
                              or (n.endswith('.fo') or n == 'wadls'))]
    return f


def publish_book(publish_path, book):
    """Copy generated files to publish_path."""

    # Assumption: The path for the book is the same as the name of directory
    # the book is in. We need to special case any exceptions.

    # Publishing directory
    book_path = publish_path

    if cfg.CONF.language:
        book_path = os.path.join(book_path, cfg.CONF.language)

    if os.path.isdir(os.path.join('target/docbkx/webhelp', book)):
        source = os.path.join('target/docbkx/webhelp', book)
    elif os.path.isdir(os.path.join('target/docbkx/webhelp/local', book)):
        source = os.path.join('target/docbkx/webhelp/local', book)
        book_path = os.path.join(book_path, 'local')
    elif os.path.isdir(os.path.join('target/docbkx/webhelp/',
                                    cfg.CONF.release_path, book)):
        source = os.path.join('target/docbkx/webhelp/',
                              cfg.CONF.release_path, book)
        book_path = os.path.join(book_path, cfg.CONF.release_path)
    elif (book in BOOK_MAPPINGS):
        source = BOOK_MAPPINGS[book]
    else:
        if cfg.CONF.debug:
            print("No build result found for book %s" % book)
        return

    if book in BOOK_PUBLISH_MAPPINGS:
        book_publish_dir = BOOK_PUBLISH_MAPPINGS[book]
    else:
        book_publish_dir = book

    book_path = os.path.join(book_path, book_publish_dir)
    if cfg.CONF.debug:
        print("Uploading book %s to %s" % (book, book_path))

    # Note that shutil.copytree does not allow an existing target directory,
    # thus delete it.
    shutil.rmtree(book_path, ignore_errors=True)

    shutil.copytree(source, book_path,
                    ignore=ignore_for_publishing)


def ensure_exists(program):
    """Check that program exists, abort if not."""
    retcode = subprocess.call(['which', program], stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE)
    if retcode != 0:
        print("Program '%s' does not exist, please install!" % program)
        sys.exit(1)


def build_book(book, publish_path, log_path):
    """Build book(s) in directory book."""

    # Note that we cannot build in parallel several books in the same
    # directory like the Install Guide. Thus we build sequentially per
    # directory.
    os.chdir(book)
    result = True
    returncode = 0
    if cfg.CONF.debug:
        print("Building in directory '%s'" % book)
    base_book = os.path.basename(book)
    base_book_orig = base_book
    comments = "-Dcomments.enabled=%s" % cfg.CONF.comments_enabled
    release = "-Drelease.path.name=%s" % cfg.CONF.release_path
    if cfg.CONF.language:
        out_filename = ("build-" + cfg.CONF.language + "-" + base_book +
                        ".log.gz")
    else:
        out_filename = "build-" + base_book + ".log.gz"
    out_file = gzip.open(os.path.join(log_path, out_filename), 'w')
    output = ""
    try:
        # Clean first and then build so that the output of all guides
        # is available
        output = subprocess.check_output(
            ["mvn", "clean"],
            stderr=subprocess.STDOUT
        )
        out_file.write(output)
        if base_book == "install-guide":
            # Build Debian
            if cfg.CONF.enable_debian_install:
                base_book = "install-guide (for debian)"
                output = subprocess.check_output(
                    ["mvn", "generate-sources", "-B",
                     comments, release,
                     "-Doperating.system=apt-debian", "-Dprofile.os=debian"],
                    stderr=subprocess.STDOUT
                )
                out_file.write(output)
            # Build Fedora
            base_book = "install-guide (for Fedora)"
            output = subprocess.check_output(
                ["mvn", "generate-sources", "-B",
                 comments, release,
                 "-Doperating.system=yum",
                 "-Dprofile.os=centos;fedora;rhel"],
                stderr=subprocess.STDOUT
            )
            out_file.write(output)
            # Build openSUSE
            base_book = "install-guide (for openSUSE)"
            output = subprocess.check_output(
                ["mvn", "generate-sources", "-B",
                 comments, release,
                 "-Doperating.system=zypper", "-Dprofile.os=opensuse;sles"],
                stderr=subprocess.STDOUT
            )
            out_file.write(output)
            # Build Ubuntu
            base_book = "install-guide (for Ubuntu)"
            output = subprocess.check_output(
                ["mvn", "generate-sources", "-B",
                 comments, release,
                 "-Doperating.system=apt", "-Dprofile.os=ubuntu"],
                stderr=subprocess.STDOUT
            )
            # Success
            base_book = "install-guide (for "
            if cfg.CONF.enable_debian_install:
                base_book += "Debian, "
            base_book += "Fedora, openSUSE, Ubuntu)"
        # HOT template guide
        elif base_book == 'hot-guide':
            # Make sure that the build dir is clean
            if os.path.isdir('build'):
                shutil.rmtree('build')
            # Generate the DN XML
            output = subprocess.check_output(
                ["make", "xml"],
                stderr=subprocess.STDOUT
            )
            out_file.write(output)
            # Generate the docbook book
            output = subprocess.check_output(
                ["openstack-dn2osdbk", "build/xml", "build/docbook",
                 "--toplevel", "book"],
                stderr=subprocess.STDOUT
            )
            out_file.write(output)
            output = subprocess.check_output(
                ["mvn", "generate-sources", comments, release, "-B"],
                stderr=subprocess.STDOUT
            )
        else:
            output = subprocess.check_output(
                ["mvn", "generate-sources", comments, release, "-B"],
                stderr=subprocess.STDOUT
            )
    except (subprocess.CalledProcessError, KeyboardInterrupt) as e:
        output = e.output
        returncode = e.returncode
        result = False

    out_file.write(output)
    out_file.close()
    if result:
        publish_book(publish_path, base_book_orig)
    return (base_book, result, output, returncode)


def is_book_master(filename):
    """Check if a file is a book master file.

    Returns True if filename is one of the special filenames used for the
    book master files.

    We do not parse pom.xml for the includes directive to determine
    the top-level files and thus have to use a heuristic.

    """

    return ((filename.startswith(('bk-', 'bk_', 'st-', 'api-'))
             and filename.endswith('.xml')) or
            filename == 'openstack-glossary.xml')


def print_unused(rootdir, ignore_dirs, included_by):
    """Print list of files that are not included anywhere."""

    if cfg.CONF.print_unused_files:
        print("Checking for files that are not included anywhere...")
        print(" Note: This only looks at files included by an .xml file "
              "but not for files included by other files like .wadl.")
        for root, dirs, files in os.walk(rootdir):
            dirs[:] = filter_dirs(dirs)

            # Filter out directories to be ignored
            if ignore_dirs:
                dirs[:] = [d for d in dirs if d not in ignore_dirs]

            for f in files:
                if not f.endswith('.xml'):
                    continue
                f_base = os.path.basename(f)
                f_abs = os.path.abspath(os.path.join(root, f))

                if (f_abs not in included_by and f_base != "pom.xml"
                   and not is_book_master(f_base)):
                    f_rel = os.path.relpath(f_abs, rootdir)
                    print("  %s " % f_rel)
        print("\n")


def generate_affected_books(rootdir, book_bk, ignore_dirs, abs_ignore_dirs,
                            included_by):
    """Generate list of affected books."""

    affected_books = set()

    # Generate list of modified_files
    # Do not select deleted files, just Added, Copied, Modified, Renamed,
    # or Type changed
    modified_files = get_modified_files(rootdir, "--diff-filter=ACMRT")
    modified_files = [os.path.abspath(f) for f in modified_files]
    if ignore_dirs:
        for idir in abs_ignore_dirs:
            non_ignored_files = []
            for f in modified_files:
                if not f.startswith(idir):
                    non_ignored_files.append(f)
            modified_files = non_ignored_files

    # 2. Find all modified files and where they are included

    # List of files that we have to iterate over, these are affected
    # by some modification
    new_files = modified_files

    # All files that are affected (either directly or indirectly)
    affected_files = set(modified_files)

    # 3. Iterate over files that have includes on modified files
    # and build a closure - the set of all files (affected_files)
    # that have a path to a modified file via includes.
    while new_files:
        new_files_to_check = new_files
        new_files = []
        for f in new_files_to_check:
            if "doc/hot-guide/" in f:
                affected_books.add('hot-guide')
                continue
            # Skip bk*.xml files
            if is_book_master(os.path.basename(f)):
                book_modified = book_bk[f]
                if book_modified not in affected_books:
                    affected_books.add(book_modified)
                continue
            if f not in included_by:
                continue
            for g in included_by[f]:
                if g not in affected_files:
                    new_files.append(g)
                    affected_files.add(g)
    return affected_books


def find_affected_books(rootdir, book_exceptions, file_exceptions,
                        force, ignore_dirs):
    """Check which books are affected by modified files.

    Returns a set with books.
    """
    book_root = rootdir

    books = []

    build_all_books = (force or check_modified_affects_all(rootdir) or
                       cfg.CONF.only_book)

    # Dictionary that contains a set of files.
    # The key is a filename, the set contains files that include this file.
    included_by = {}

    # Dictionary with books and their bk*.xml files
    book_bk = {}

    # List of absolute paths of ignored directories
    abs_ignore_dirs = []
    # 1. Iterate over whole tree and analyze include files.
    # This updates included_by, book_bk and books.
    for root, dirs, files in os.walk(rootdir):
        dirs[:] = filter_dirs(dirs)

        # Filter out directories to be ignored
        if ignore_dirs:
            for d in dirs:
                if d in ignore_dirs:
                    abs_ignore_dirs.append(
                        os.path.join(os.path.join(root, d)))
            dirs[:] = [d for d in dirs if d not in ignore_dirs]

        if os.path.basename(root) in book_exceptions:
            break
        # Do not process files in doc itself or top-level directory
        elif root.endswith('doc') or root == rootdir:
            continue
        elif ("pom.xml" in files and (not cfg.CONF.only_book or
                                      os.path.basename(root) in
                                      cfg.CONF.only_book)):
                books.append(root)
                book_root = root

        # No need to check single books if we build all, we just
        # collect list of books
        if build_all_books:
            continue

        for f in files:
            f_base = os.path.basename(f)
            f_abs = os.path.abspath(os.path.join(root, f))
            if is_book_master(f_base):
                book_bk[f_abs] = book_root
            if not is_testable_xml_file(f, file_exceptions):
                continue

            try:
                doc = etree.parse(f_abs)
            except etree.XMLSyntaxError as e:
                print("  Warning: file %s is invalid XML: %s" % (f_abs, e))
                continue
            for node in doc.findall(
                    '//{http://docbook.org/ns/docbook}imagedata'):
                href = node.get('fileref')
                href_abs = os.path.abspath(os.path.join(root, href))
                if href_abs in included_by:
                    included_by[href_abs].add(f_abs)
                else:
                    included_by[href_abs] = set([f_abs])

            ns = {"xi": "http://www.w3.org/2001/XInclude"}
            for node in doc.xpath('//xi:include', namespaces=ns):
                href = node.get('href')
                href_abs = os.path.abspath(os.path.join(root, href))
                if href_abs in included_by:
                    included_by[href_abs].add(f_abs)
                else:
                    included_by[href_abs] = set([f_abs])

            ns = {"wadl": "http://wadl.dev.java.net/2009/02"}
            for node in doc.xpath('//wadl:resource', namespaces=ns):
                href = node.get('href')
                hash_sign = href.rfind('#')
                if hash_sign != -1:
                    href = href[:hash_sign]
                href_abs = os.path.abspath(os.path.join(root, href))
                if href_abs in included_by:
                    included_by[href_abs].add(f_abs)
                else:
                    included_by[href_abs] = set([f_abs])
            for node in doc.xpath('//wadl:resources', namespaces=ns):
                href = node.get('href')
                # wadl:resources either have a href directly or a child
                # wadl:resource that has a href. So, check that we have
                # a href.
                if href:
                    hash_sign = href.rfind('#')
                    if hash_sign != -1:
                        href = href[:hash_sign]
                    href_abs = os.path.abspath(os.path.join(root, href))
                    if href_abs in included_by:
                        included_by[href_abs].add(f_abs)
                    else:
                        included_by[href_abs] = set([f_abs])

    print_unused(rootdir, ignore_dirs, included_by)

    if cfg.CONF.only_book:
        print("Building specified books.")
    elif build_all_books:
        print("Building all books.")
    else:
        affected_books = generate_affected_books(rootdir, book_bk, ignore_dirs,
                                                 abs_ignore_dirs, included_by)
        if affected_books:
            books = affected_books
        else:
            print("No books are affected by modified files. "
                  "Building all books.")

    return books


def build_affected_books(rootdir, book_exceptions, file_exceptions,
                         force=False, ignore_dirs=None,
                         ignore_books=None):
    """Build all the books which are affected by modified files.

    Looks for all directories with "pom.xml" and checks if a
    XML file in the directory includes a modified file. If at least
    one XML file includes a modified file the method calls
    "mvn clean generate-sources" in that directory.

    This will throw an exception if a book fails to build
    """

    if ignore_dirs is None:
        ignore_dirs = []

    books = find_affected_books(rootdir, book_exceptions,
                                file_exceptions, force, ignore_dirs)

    if ignore_books:
        books = [b for b in books if os.path.basename(b) not in ignore_books]

    # Remove cache content which can cause build failures
    shutil.rmtree(os.path.expanduser("~/.fop"),
                  ignore_errors=True)

    maxjobs = multiprocessing.cpu_count()
    # Jenkins fails sometimes with errors if too many jobs run, artificially
    # limit to 4 for now.
    # See https://bugs.launchpad.net/openstack-manuals/+bug/1221721
    if maxjobs > 4:
        maxjobs = 4
    pool = multiprocessing.Pool(maxjobs)
    print("Queuing the following books for building:")
    publish_path = get_publish_path()
    log_path = get_gitroot()
    first_book = True

    # First show books
    for book in sorted(books):
        print("  %s" % os.path.basename(book))
    print("Building all queued %d books now..." % len(books))

    # And then queue - since we wait for the first book to finish.
    try:
        for book in sorted(books):
            if cfg.CONF.debug or not cfg.CONF.parallel:
                (book, result, output, retcode) = build_book(book,
                                                             publish_path,
                                                             log_path)
                logging_build_book([book, result, output, retcode])
            else:
                res = pool.apply_async(build_book,
                                       (book, publish_path, log_path),
                                       callback=logging_build_book)
                if first_book:
                    first_book = False
                    # The first invocation of maven might download loads of
                    # data locally, we cannot do this in parallel. So, wait
                    # here for the first job to finish before running further
                    # mvn jobs.
                    res.get()
        pool.close()
        pool.join()
    except KeyboardInterrupt:
        pool.terminate()
        pool.join()

    any_failures = False
    for book, result, _, _ in sorted(RESULTS_OF_BUILDS,
                                     key=operator.itemgetter(0)):
        if result:
            print(">>> Build of book %s succeeded." % book)
        else:
            any_failures = True

    if cfg.CONF.create_index:
        index.generate_index_file(get_publish_path())

    if any_failures:
        for book, result, output, returncode in RESULTS_OF_BUILDS:
            if not result:
                print(">>> Build of book %s failed (returncode = %d)."
                      % (book, returncode))
                print("\n%s" % output)

        print("Building of books finished with failures.\n")
        return 1
    else:
        print("Building of books finished successfully.\n")

    if len(RESULTS_OF_BUILDS) != len(books):
        print("ERROR: %d queued for building but only %d build!" %
              (len(books), len(RESULTS_OF_BUILDS)))
        return 1
    return 0


def add_exceptions(file_exception, verbose):
    """Add list of exceptions from file_exceptions."""

    for entry in file_exception:
        if verbose:
            print(" Adding file to ignore list: %s" % entry)
        FILE_EXCEPTIONS.append(entry)


def add_build_exceptions(build_file_exception, verbose):
    """Add list of exceptions from build_file_exceptions."""

    for entry in build_file_exception:
        if verbose:
            print(" Adding file to build ignore list: %s" % entry)
        BUILD_FILE_EXCEPTIONS.append(entry)


def add_url_exceptions(url_exception, verbose):
    """Add list of exceptions from url_exception."""

    for entry in url_exception:
        if verbose:
            print(" Adding URL to ignore list: %s" % entry)
        URL_EXCEPTIONS.append(entry)


cli_OPTS = [
    cfg.BoolOpt("api-site", default=False,
                help="Enable special handling for api-site repository."),
    cfg.BoolOpt('check-all', default=False,
                help="Run all checks."),
    cfg.BoolOpt('check-build', default=False,
                help="Check building of books using modified files."),
    cfg.BoolOpt("check-deletions", default=False,
                help="Check that deleted files are not used."),
    cfg.BoolOpt("check-links", default=False,
                help="Check that linked URLs are valid and reachable."),
    cfg.BoolOpt("check-niceness", default=False,
                help="Check the niceness of files, for example whitespace."),
    cfg.BoolOpt("check-syntax", default=False,
                help="Check the syntax of modified files."),
    cfg.BoolOpt("create-index", default=True,
                help="When publishing create an index.html file to find books "
                "in an easy way."),
    cfg.BoolOpt('force', default=False,
                help="Force the validation of all files "
                "and build all books."),
    cfg.BoolOpt("parallel", default=True,
                help="Build books in parallel (default)."),
    cfg.BoolOpt("print-unused-files", default=False,
                help="Print list of files that are not included anywhere as "
                "part of check-build."),
    cfg.BoolOpt("publish", default=False,
                help="Setup content in publish-docs directory for "
                "publishing to external website."),
    cfg.MultiStrOpt("build-file-exception",
                    help="File that will be skipped during delete and "
                         "build checks to generate dependencies. This should "
                         "be done for invalid XML files only."),
    cfg.MultiStrOpt("file-exception",
                    help="File that will be skipped during niceness and "
                         "syntax validation."),
    cfg.MultiStrOpt("ignore-dir",
                    help="Directory to ignore for building of manuals. The "
                         "parameter can be passed multiple times to add "
                         "several directories."),
    cfg.MultiStrOpt("ignore-book",
                    help="Book to ignore when building manuals. The "
                         "parameter can be passed multiple times to add "
                         "several books."),
    cfg.StrOpt('language', default=None, short='l',
               help="Build translated manual for language in path "
               "generate/$LANGUAGE ."),
    cfg.MultiStrOpt('only-book', default=None,
                    help="Build each specified manual."),
    cfg.MultiStrOpt("url-exception",
                    help="URL that will be skipped during reachability "
                         "check."),
    cfg.BoolOpt("enable-debian-install", default=False,
                help="Enable building of Debian Install Guide."),
]

OPTS = [
    # NOTE(jaegerandi): books, target-dirs, publish-dir could be a
    # DictOpt but I could not get this working properly.
    cfg.MultiStrOpt("book", default=None,
                    help="Name of book that needs special mapping. "
                    "This is the name of directory where the pom.xml "
                    "file lives."),
    cfg.MultiStrOpt("target-dir", default=None,
                    help="Directory name in target dir for a book. "
                    "The option must be in the same order as the book "
                    "option."),
    cfg.MultiStrOpt("publish-dir", default=None,
                    help="Directory name where book will be copied to "
                    "in publish-docs directory. This option must be in "
                    "same order as the book option. Either give this option "
                    "for all books or for none. If publish-dir is not "
                    "specified, book is used as publish-dir."),
    cfg.StrOpt("repo-name", default=None,
               help="Name of repository."),
    cfg.StrOpt("release-path", default="trunk",
               help="Value to pass to maven for release.path.name."),
    cfg.StrOpt("comments-enabled", default="0",
               help="Value to pass to maven for comments.enabled."),
]


def handle_options():
    """Setup configuration variables from config file and options."""

    CONF = cfg.CONF
    CONF.register_cli_opts(cli_OPTS)
    CONF.register_opts(OPTS)

    default_config_files = [os.path.join(get_gitroot(), 'doc-test.conf')]
    CONF(sys.argv[1:], project='documentation', prog='openstack-doc-test',
         version=os_doc_tools.__version__,
         default_config_files=default_config_files)

    if CONF.repo_name:
        print("Testing repository '%s'\n" % CONF.repo_name)

    if CONF.verbose:
        print("Verbose execution")

    if CONF.debug:
        print("Enabling debug code")

    if CONF.language:
        print("Building for language '%s'" % CONF.language)

    if CONF.file_exception:
        add_exceptions(CONF.file_exception, CONF.verbose)

    if CONF.build_file_exception:
        add_build_exceptions(CONF.build_file_exception, CONF.verbose)

    if CONF.url_exception:
        add_url_exceptions(CONF.url_exception, CONF.verbose)

    if (not CONF.check_build and not CONF.check_deletions and
       not CONF.check_niceness and not CONF.check_syntax and
       not CONF.check_links):
        CONF.check_all = True

    if CONF.check_all:
        CONF.check_deletions = True
        CONF.check_build = True
        CONF.check_links = True
        CONF.check_niceness = True
        CONF.check_syntax = True

    if CONF.publish:
        CONF.create_index = False

    if CONF.check_build and CONF.book and CONF.target_dir:
        if len(CONF.book) != len(CONF.target_dir):
            print("ERROR: book and target_dir options need to have a 1:1 "
                  "relationship.")
            sys.exit(1)
        if (CONF.publish_dir and
           len(CONF.publish_dir) != len(CONF.target_dir)):
            print("ERROR: publish_dir and target_dir need to have a 1:1 "
                  "relationship if publish_dir is specified.")
            sys.exit(1)
        for i in range(len(CONF.book)):
            BOOK_MAPPINGS[CONF.book[i]] = CONF.target_dir[i]
            if CONF.verbose:
                print(" Target dir for %s is %s" %
                      (CONF.book[i], BOOK_MAPPINGS[CONF.book[i]]))
        if CONF.publish_dir:
            for i in range(len(CONF.book)):
                BOOK_PUBLISH_MAPPINGS[CONF.book[i]] = CONF.publish_dir[i]
                if CONF.verbose:
                    print(" Publish dir for %s is %s" %
                          (CONF.book[i], BOOK_PUBLISH_MAPPINGS[CONF.book[i]]))

    if CONF.check_build:
        if CONF.verbose:
            if cfg.CONF.ignore_dir:
                for d in cfg.CONF.ignore_dir:
                    print(" Ignore directory: %s" % d)
            print(" Release path: %s" % cfg.CONF.release_path)
            print(" Comments enabled: %s" % cfg.CONF.comments_enabled)


def check_no_building():
    """Figure out whether we need to build XML files."""

    CONF = cfg.CONF
    if not CONF.force and www_touched():
        print("Only files in www directory changed, nothing to do.\n")
        return True
    # Build everything if we publish so that regularly all manuals are
    # published, for testing ignore the locale directories.
    if (not CONF.force and not CONF.publish and not CONF.language
            and only_po_touched()):
        print("Only files in locale directories changed, nothing to do.\n")
        return True
    # If only RST files are touched, there's nothing to do.
    if (not CONF.force and only_rst_touched()):
        print("Only RST files changed, nothing to do.\n")
        return True
    return False


def doctest():
    """Central entrypoint, parses arguments and runs tests."""

    start_time = time.time()
    CONF = cfg.CONF
    print("\nOpenStack Doc Checks (using openstack-doc-tools version %s)\n"
          % os_doc_tools.__version__)
    try:
        handle_options()
    except cfg.Error as e:
        print(e)
        return 1
    print_gitinfo()
    errors = 0

    doc_path = get_gitroot()
    if CONF.language:
        doc_path = os.path.join(doc_path, 'generated', CONF.language)
        if CONF.verbose:
            print("Using %s as root" % doc_path)
    elif not CONF.api_site:
        doc_path = os.path.join(doc_path, 'doc')

    if check_no_building():
        if CONF.create_index:
            index.generate_index_file(get_publish_path())
        return

    if CONF.check_syntax or CONF.check_niceness or CONF.check_links:
        if CONF.force:
            errors += validate_all_files(doc_path, FILE_EXCEPTIONS,
                                         CONF.verbose,
                                         CONF.check_syntax,
                                         CONF.check_niceness,
                                         CONF.check_links,
                                         CONF.api_site)
        else:
            errors += validate_modified_files(doc_path, FILE_EXCEPTIONS,
                                              CONF.verbose,
                                              CONF.check_syntax,
                                              CONF.check_niceness,
                                              CONF.check_links,
                                              CONF.api_site)

    if CONF.check_deletions:
        errors += check_deleted_files(doc_path, BUILD_FILE_EXCEPTIONS,
                                      CONF.verbose)

    if CONF.check_build:
        # Some programs are called in subprocesses,  make sure that they
        # really exist.
        ensure_exists("mvn")
        errors += build_affected_books(doc_path, BOOK_EXCEPTIONS,
                                       BUILD_FILE_EXCEPTIONS,
                                       CONF.force,
                                       CONF.ignore_dir,
                                       CONF.ignore_book)

    elapsed_time = (time.time() - start_time)
    print("Run time was: %.2f seconds." % elapsed_time)
    if errors == 0:
        print("Congratulations, all tests passed!")
        return 0
    else:
        print("Some tests failed!")
        return 1

if __name__ == "__main__":
    sys.exit(doctest())