mirrorkit 0.2.1 / usr / bin / mirrorkit

#!/usr/bin/python3
# -*- coding: utf-8 -*-

# Copyright (C) 2008-2013 Stéphane Graber
# Author: Stéphane Graber <stgraber@ubuntu.com>

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You can find the license on Debian systems in the file
# /usr/share/common-licenses/GPL-2

import argparse
import configparser
import logging
import os
import string
import subprocess
import sys
import tempfile
import time
import urllib.parse


def get_path_size(path):
    total_size = 0
    for dirpath, dirnames, filenames in os.walk(path):
        for f in filenames:
            fp = os.path.join(dirpath, f)
            total_size += os.path.getsize(fp)
    return total_size


def parse_config(path):
    """
        Return a dict representation of a .ini config
    """
    config = {}

    configp = configparser.ConfigParser()
    try:
        configp.read(path)
    except:
        return config

    for section in configp.sections():
        config_section = {}
        for option in configp.options(section):
            value = configp.get(section, option)
            if ", " in value:
                value = [entry.strip('"').strip()
                         for entry in value.split(", ")]
            else:
                value = value.strip('"').strip()
            config_section[option] = value
        config[section] = config_section

    return config


def load_config(config_path):
    """
        Read an ini configuration file and return a MirrorKitConfig object.
    """

    # Get a dict representation of the config
    config = parse_config(config_path)

    # Process the global section
    settings = {}
    if not "global" in config:
        logging.error("Missing 'global' config section.")
        return None

    if not config['global'].get("publish_path", None):
        logging.error("Missing 'publish_path' value.")
        return None

    settings['publish_path'] = config['global']['publish_path']
    settings['log_path'] = config['global'].get("log_path", None)
    if settings['log_path']:
        if not config['global'].get("log_template_path", None):
            logging.error("Missing 'log_template_path' value.")
            return None
        settings['log_template_path'] = config['global'].get(
            "log_template_path", None)
    settings['apache_conf_path'] = config['global'].get("apache_conf_path",
                                                        None)

    settings['http_base'] = config['global'].get("http_base", "/")

    settings['mirrors'] = []
    for mirror_name in config['global'].get("mirrors", []):
        mirror = {}
        mirror['name'] = mirror_name

        if mirror_name not in config:
            logging.error("Couldn't find settings for mirror: %s" %
                          mirror_name)
            return None

        mirror['source'] = config[mirror_name].get("source", None)
        mirror['sources'] = config[mirror_name].get("sources", False) == "true"

        for key in ("pockets", "components", "sub-components",
                    "architectures"):
            value = config[mirror_name].get(key, None)

            if not isinstance(value, list):
                value = [value]

            mirror[key] = value

        # Expand sub-components
        extra_components = []
        for entry in mirror['sub-components']:
            for component in mirror['components']:
                extra_components.append("%s/%s" % (component, entry))
        mirror['components'] += extra_components
        mirror.pop("sub-components")

        for key in ("source", "pockets", "components", "architectures"):
            if not mirror[key]:
                logging.error("Missing value for '%s' in mirror: %s" %
                              (key, mirror_name))
                return None

        settings['mirrors'].append(type("MirrorKitMirror", (object,), mirror))

    # Create our fake object
    return type("MirrorKitConfig", (object,), settings)


def debmirror_command(config, mirror):
    """
        Generate the appropriate debmirror command.
    """

    url = urllib.parse.urlparse(mirror.source)

    if not url:
        logging.error("Invalid URL: %s" % url)
        return None

    if url.scheme not in ("http", "https", "ftp", "rsync"):
        logging.error("Invalid URL scheme: %s" % url.scheme)
        return None

    cmd = ["debmirror", "-v",
           "--host=%s" % url.netloc,
           "--root=%s" % url.path,
           "--arch=%s" % ",".join(mirror.architectures),
           "--dist=%s" % ",".join(mirror.pockets),
           "--section=%s" % ",".join(mirror.components),
           "--progress",
           "--method=%s" % url.scheme,
           "--ignore-release-gpg"]

    if not mirror.sources:
        cmd += ["--nosource"]

    cmd += [os.path.join(config.publish_path, mirror.name)]

    return cmd


def run_debmirror(config, mirror, log):
    """
        Run debmirror.
        Output is written to stdout and stderr.
    """

    cmd = debmirror_command(config, mirror)
    if not cmd:
        return None

    if subprocess.call(cmd, stdout=log, stderr=log,
                       universal_newlines=True) != 0:
        logging.error("debmirror failed to run for: %s" % mirror.name)
        return None

    return (True, cmd)


def generate_report(config, mirror, success, log):
    log_filename = "%s.%s.html" % (mirror.name,
                                   time.strftime("%Y%m%d.%H-%M-%S",
                                                 time.gmtime()))
    log_file = os.path.join(config.log_path, log_filename)

    log.seek(0)

    variables = {'date': time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()),
                 'name': mirror.name,
                 'status': "Success" if success else "Failure",
                 'status_html': "<span id=\"success\">Success</span>"
                                if success else
                                "<span id=\"failure\">Failure</span>",
                 'size': round(get_path_size(os.path.join(
                     config.publish_path, mirror.name)) / 1048576, 2),
                 'source': mirror.source,
                 'destination': os.path.join(config.publish_path, mirror.name),
                 'pockets': ", ".join(mirror.pockets),
                 'components': ", ".join(mirror.components),
                 'architectures': ", ".join(mirror.architectures),
                 'sources': "yes" if mirror.sources else "no",
                 'command': " ".join(debmirror_command(config, mirror)),
                 'log': log.read()}

    # Generate the html file
    with open(log_file, "w+") as log_fd:
        with open(config.log_template_path, "r") as fd:
            template_str = fd.read()
        template = string.Template(template_str)
        log_fd.write(template.safe_substitute(variables))

    # Create symlink
    log_symlink = os.path.join(config.log_path, "%s.html" % mirror.name)
    if os.path.exists(log_symlink):
        os.remove(log_symlink)
    os.symlink(log_filename, log_symlink)


def generate_apache_conf(config):
    if not os.path.exists(os.path.dirname(config.apache_conf_path)):
        logging.info("Apache configuration directory doesn't exist, skipping")
        return

    with open(config.apache_conf_path, "w+") as fd:
        for mirror in config.mirrors:
            fd.write("Alias %s %s\n" % (os.path.join(config.http_base,
                                                     mirror.name),
                                        os.path.join(config.publish_path,
                                                     mirror.name)))

        if config.log_path:
            fd.write("Alias %s %s\n" % (os.path.join(config.http_base, "logs"),
                                        config.log_path))
            fd.write("""
<Location %s>
    Options Indexes FollowSymLinks MultiViews
    <IfVersion < 2.3 >
        Order allow,deny
        Allow from all
    </IfVersion>
    <IfVersion >= 2.3>
        Require all granted
    </IfVersion> 
</Location>
""" % (os.path.join(config.http_base, "logs")))

        for mirror in config.mirrors:
            rel_path = os.path.join(config.http_base, mirror.name)
            rel_path_escaped = rel_path.replace("/", "\\/")

            fd.write("""
<Location %s>
    RewriteEngine On
    RewriteCond %%{REQUEST_FILENAME} !-f
    RewriteCond %%{REQUEST_FILENAME} !-d
    RewriteRule .*%s\/pool\/(.*) %s.orig/pool/$1 [L]
    RewriteRule .*%s\/dists\/(.*) %s.orig/dists/$1 [L]

    Options Indexes FollowSymLinks MultiViews
    <IfVersion < 2.3 >
        Order allow,deny
        Allow from all
    </IfVersion>
    <IfVersion >= 2.3>
        Require all granted
    </IfVersion> 
</Location>
<Location %s/project>
    Deny from all
</Location>
ProxyPass %s.orig/ %s/
""" % (rel_path,
       rel_path_escaped, rel_path,
       rel_path_escaped, rel_path,
       rel_path,
       rel_path, mirror.source))

        fd.write("ProxyRequests off")


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="mirrorkit")
    parser.add_argument("--config", metavar="CONFIG",
                        help="Path to the configuration file",
                        default="/etc/mirrorkit.conf")
    args = parser.parse_args()

    # Basic checks
    if not os.path.exists(args.config):
        parser.error("Configuration file doesn't exist: %s" % args.config)
        sys.exit(1)

    # Load the configuration
    config = load_config(args.config)
    if not config:
        sys.error(1)

    if not config.mirrors:
        sys.exit(0)

    # Check if template exists
    if config.log_template_path:
        if not os.path.exists(config.log_template_path):
            parser.error("Missing log template: %s" % config.log_template_path)
            sys.exit(1)

    # Create any missing path
    if not os.path.exists(config.publish_path):
        logging.debug("Creating missing path: %s" % config.publish_path)
        os.makedirs(config.publish_path)

    if config.log_path and not os.path.exists(config.log_path):
        logging.debug("Creating missing path: %s" % config.log_path)
        os.makedirs(config.log_path)

    # Start the mirroring
    for mirror in config.mirrors:
        logging.info("Beginning to mirror: %s" % mirror.name)

        log_fd, log_path = tempfile.mkstemp()
        log = os.fdopen(log_fd)

        retval = run_debmirror(config, mirror, log)
        logging.info("Done mirroring: %s" % mirror.name)

        if config.log_path:
            logging.info("Generating html report for: %s" % mirror.name)
            generate_report(config, mirror, retval is not None, log)

        log.close()
        os.remove(log_path)

    # Generate the http configuration
    if config.apache_conf_path:
        logging.info("Generating apache2 configuration: %s" %
                     config.apache_conf_path)
        generate_apache_conf(config)