mic2 0.24.12-1 / usr / bin / mic-image-creator

#!/usr/bin/python -t
#
# mic-image-creator : Creates all kinds of image
#
# Copyright 2009, Intel Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

from __future__ import with_statement
import os
import os.path
import sys
import time
import optparse
import logging
import shutil
import subprocess
import string
import tempfile
import traceback
import re
import glob

import mic.appcreate as appcreate
import mic.imgcreate as imgcreate
import mic.chroot as mychroot
import mic.utils as utils

import signal
def sig_interrupt(signum=None, frame=None):
    raise imgcreate.CreatorError('Canceled by ^C.')
signal.signal(signal.SIGINT, sig_interrupt)

try:
    import mic.__version__
    version = mic.__version__.version
except:
    version = 'unknown'

class FatalError(Exception):
    def __init__(self, msg = None, no_error = False):
        self.msg = msg

    def __str__(self):
        return self.msg

parser = None

DEFAULT_TMPDIR="/var/tmp"
DEFAULT_CACHEDIR="~/.mic2_cachedir"

def parse_options(args):
    global parser
    parser = optparse.OptionParser(version=version)

    commonopt = optparse.OptionGroup(parser, "General options",
                                  "These options define the created image.")

    commonopt.add_option("-c", "--config", type="string", dest="config",
                    help="Path to kickstart config file")

    commonopt.add_option("-f", "--format", type="string", dest="format",
                    help="Available image formats: %s" % (", ".join(imgcreate.supported_formats + appcreate.supported_formats)))

    parser.add_option_group(commonopt)

    # options related to the config of your system
    sysopt = optparse.OptionGroup(parser, "System directory options",
                                  "These options define directories used on your system for creating the image")
    sysopt.add_option("-t", "--tmpdir", type="string",
                      dest="tmpdir",
                      help="Temporary directory to use (default: "+DEFAULT_TMPDIR+", if not found from siteconf)")
    sysopt.add_option("-k", "--cache", type="string",
                      dest="cachedir", default=None,
                      help="Cache directory to use (default: "+DEFAULT_CACHEDIR+", if not found from siteconf)")
    sysopt.add_option("-o", "--outdir", type="string",
                      dest="outdir", default=None,
                      help="Output directory to use (default: current work dir)")
    sysopt.add_option("", "--release", type="string",
                      dest="release", default=None,
                      help="Generate a release with all necessary files for publishing.")
    sysopt.add_option("", "--genchecksum", action="store_true",
                      dest="genchecksum", default=False,
                      help="Generate checksum for image file if this option is provided")
    sysopt.add_option("-P", "--prefix", type="string",
                      dest="prefix", default=None,
                      help="Image name prefix (default: lowercase distro_name from siteconfig or meego if not provided)")
    sysopt.add_option("-S", "--suffix", type="string",
                      dest="suffix", default=None,
                      help="Image name suffix (default: date stamp)")
    sysopt.add_option("-b", "--build-bootstrap", action="store_true",
                      dest="buildbootstrap", default=False,
                      help="Build a bootstrap, it should be used with -B or --bootstrap together")
    sysopt.add_option("", "--rebuild-bootstrap", action="store_true", default=False,
                      help="Rebuild the bootstrap")
    sysopt.add_option("", "--mainrepo", type="string",
                      dest="mainrepo", default=None,
                      help="Specify main repo name, it must be a repo name in your kickstart file")
    sysopt.add_option("", "--siteconf", type="string", action="store",
                      dest="siteconf",
                      help="Specify site config file, it can be used to set tmpdir, cachedir, outputdir, "
                      "it also can be used to remap/reroute a repo in kickstart to another equivalent repo, "
                      "the default site configs are "+utils.misc.DEFAULT_SITECONF_GLOBAL+" "
                      "and "+utils.misc.DEFAULT_SITECONF_USER+".")
    sysopt.add_option("", "--ignore-siteconf", action="store_true",
                      dest="ignore_siteconf", default=False,
                      help="Ignore site configs, this will ignore the default site configs, also ignore --siteconf option.")
    sysopt.add_option("", "--repo", type="string", action="append", metavar="REPO",
                      dest="addrepos",
                      help="Specify additional repo, it may be a simple URL or a complicated tuple string, you can specify multiple repos using this option multiple times")
    sysopt.add_option("", "--default-ks", type="string",
                      help="Set which kickstart file from repos to select in advance, if specified, there isn't interaction needed any more.")
    sysopt.add_option("", "--pkgmgr", type="string",
                      help="Specify a package manager, the available package managers have zypper and yum currently.")
    sysopt.add_option("-B", "--bootstrap", type="string",
                      dest="bootstrap", default=None,
                      help="Use a given bootstrap env to create image")
    sysopt.add_option("-a", "--arch", type="string",
                      dest="arch", default=None,
                      help="Specify target arch of image, for example: arm")
    sysopt.add_option("", "--save-kernel", action="store_true",
                      dest="save_kernel", default=False,
                      help="Save kernel image file into outdir")
    sysopt.add_option("", "--use-comps", action="store_true",
                      dest="use_comps", default=False,
                      help="Use comps instead of patterns if comps exists")
    sysopt.add_option("", "--run-mode", type="int",
                      dest="run_mode", default=-1,
                      help="Specify run mode, 0 means legacy, 1 means bootstrap, by default it is gotten from siteconfig file.")
    sysopt.add_option("", "--alt-initrd-name", type="string",
                      dest="alt_initrd_name", default=None,
                      help="Name of alternate initrd image, if NOT to use the one from kernel package")
    sysopt.add_option("", "--record-pkgs", type="string",
                      dest="record_pkgs", default=None,
                      help="Record the installed packages, valid values: name, content")
    sysopt.add_option("", "--local-pkgs-path", type="string",
                      dest="local_pkgs_path", default=None,
                      help="Path for local pkgs (rpms) to be installed")
    sysopt.add_option("", "--include-source", action="store_true",
                      dest="include_src", default=False,
                      help="Generate a image with source rpms included")

    parser.add_option_group(sysopt)

    # options related to live USB
    liveusbopt = optparse.OptionGroup(parser, "Live USB file image options",
                                  "These options are for creating the live USB file image")
    liveusbopt.add_option("-i", "--interactive", action="store_true",
                      dest="interactive", default=False,
                      help="Directly write into a USB disk.")
    liveusbopt.add_option("", "--fstype", type="string",
                      dest="fstype", default="vfat",
                      help="File system type for live USB file image, ext3 or vfat, the default is vfat.")
    liveusbopt.add_option("", "--overlay-size-mb", type="int", default=64,
                      help="Overlay size in MB as unit (size of changes you can save in your live USB disk.)")
    liveusbopt.add_option("", "--volumeid", type="string", default=None,
                      help="Specify volume id, valid only for livecd")
    parser.add_option_group(liveusbopt)

    # options related to nand image
    nandopt = optparse.OptionGroup(parser, "MRST NAND image options",
                                  "These options are for creating the MRST NAND image")
    nandopt.add_option("", "--initrd-url", type="string",
                      dest="initrd_url", default=None,
                      help="Remote URL of INITRD image.")
    nandopt.add_option("", "--initrd-path", type="string",
                      dest="initrd_path", default=None,
                      help="Local path of INITRD image.")
    nandopt.add_option("", "--kernel-url", type="string",
                      dest="kernel_url", default=None,
                      help="Remote URL of kernel file (vmlinuz).")
    nandopt.add_option("", "--kernel-path", type="string",
                      dest="kernel_path", default=None,
                      help="Local path of kernel file (vmlinuz).")
    nandopt.add_option("", "--kernel-rpm-url", type="string",
                      dest="kernel_rpm_url", default=None,
                      help="Remote URL of kernel rpm package.")
    nandopt.add_option("", "--kernel-rpm-path", type="string",
                      dest="kernel_rpm_path", default=None,
                      help="Local path of kernel rpm package.")

    nandopt.add_option("", "--bootimg-only", action="store_true", dest="bootimg_only",
                      help="Create boot image only.")
    parser.add_option_group(nandopt)

    # Don't compress the image.
    sysopt.add_option("-s", "--skip-compression", action="store_true", dest="skip_compression",
                      default=False, help=optparse.SUPPRESS_HELP)
    parser.add_option("", "--skip-minimize", action="store_true", dest="skip_minimize",
                      default=False, help=optparse.SUPPRESS_HELP)
    sysopt.add_option("-p", "--package", type="string", dest="package", default="none",
                      help="Package format, this option will package up output into a given-format package, "
                           "currently only tar, tar.gz and tar.bz2 are supported, the default "
                           "is none, i.e. doesn't package up output")
    sysopt.add_option("", "--compress-disk-image", type="string",
                     dest="compress_disk_image", default=None,
                     help="Compress the disk image that is created. When using --release option default is bz2, otherwise default is none. Supported compression methods: bz2, none")
    parser.add_option("", "--traceback", action="store_true",
                      default=False, help="Dump call stack")

    # Start a shell in the chroot for post-configuration.
    parser.add_option("", "--image-info", type="string",
                      dest="image_info", default=None,
                      help=optparse.SUPPRESS_HELP)
    parser.add_option("", "--pipe", type="string",
                      dest="pipe", default=None,
                      help=optparse.SUPPRESS_HELP)
    parser.add_option("-l", "--shell", action="store_true", dest="give_shell",
                      help=optparse.SUPPRESS_HELP)

    utils.setup_logging(parser)

    logging.getLogger().setLevel(logging.FATAL)
    (options, args) = parser.parse_args()
    if logging.getLogger().level in (logging.DEBUG, logging.INFO, logging.WARN) or options.traceback:
        logging.getLogger().setLevel(logging.NOTSET)

    return options


def run_in_bootstrap(bootstrap, args, bindmounts, arch = None):
    def chroot_bootstrap():
        """Chroot into bootstrap env"""
        try:
            os.chroot(bootstrap)
            os.chdir("/")
        except:
           raise FatalError("Please specify a workable bootstrap env.")

    globalmounts = mychroot.setup_chrootenv(bootstrap, bindmounts)

    logging.info("Starting bootstrap in dir '%s' with arch '%s' and arguments '%s'." % (bootstrap,arch,args))

    try:
        if arch and arch.startswith("arm"):
            imgcreate.setup_qemu_emulator(bootstrap, "arm")
        sys.stdout.flush()
        utils.misc.set_proxy_environ()
        ret = subprocess.call(args, preexec_fn = chroot_bootstrap)
        utils.misc.unset_proxy_environ()
    finally:
        sys.stdout.flush()
        mychroot.cleanup_chrootenv(bootstrap, bindmounts, globalmounts)

    if ret != 0:
        raise FatalError("Failed to run mic inside bootstrap environment. Return code: %s" % (ret))

def get_siteconf_diropt(siteconf, option):
    if not siteconf:
        return None
    if option:
        if siteconf.has_section("main") and siteconf.has_option("main", option):
            value = siteconf.get("main", option)
            if value:
                try:
                    imgcreate.fs.makedirs(value)
                except:
                    raise FatalError("Error: invalid tmpdir name %s" % value)
                return os.path.abspath(os.path.expanduser(value))
    return None

def get_siteconf_opt(siteconf, section, option):
    if not siteconf or not section or not option:
        return None

    if siteconf.has_section(section) and siteconf.has_option(section, option):
        value = siteconf.get(section, option)
        if value:
            return value

    return None

def get_options_from_siteconf(siteconf, options):
    ignore_siteconf = get_siteconf_opt(siteconf, "main", "ignore_siteconf")
    if ignore_siteconf and ignore_siteconf != "0":
        return
    optlist = options.__dict__
    for key in optlist.keys():
        value = getattr(options, key)
        if key == "run_mode" and value == -1:
            """ If run_mode == -1, we should read it from siteconf """
            value = None
        if value == None or (type(value) is bool and value == False) or value == "":
            value = get_siteconf_opt(siteconf, "main", key)
            if key in ("run_mode", "overlay_size_mb"):
                if value and value.isdigit():
                    setattr(options, key, int(value))
                continue
            if key in ("compress_disk_image"):
                """ none is also valid value for it """
                setattr(options, key, value)
                continue
            if value in (None, "0", "none"):
                continue
            elif value == "1":
                setattr(options, key, True)
            else:
                setattr(options, key, value)

def rebuild_argv(argv, options, logfile = None):
    blacklist = ["bootstrap", "buildbootstrap", "rebuild_bootstrap", "mainrepo", "repo", "default_ks", "siteconf", "ignore_siteconf", "run_mode"]
    changed_opts = {"overlaysizemb":"overlay-size-mb", "cachedir":"cache", "give_shell":"shell",
                    "use_comps":"use-comps", "run_mode":"run-mode", "save_kernel":"save-kernel",
                    "file":"logfile", "include_src":"include-source"
                   }
    myargv = []
    myargv.append("/usr/bin/mic-image-creator")

    optlist = options.__dict__
    for key in optlist.keys():
        if optlist[key]:
            if key in blacklist:
                continue
            value = getattr(options, key)
            if value == None or (type(value) is bool and value == False) or value == "":
                continue
            optstr = "--%s" % key.replace("_", "-")
            if key in changed_opts.keys():
                optstr = "--%s" % changed_opts[key]
            if type(value) is not bool:
                optstr += "=%s" % value
            myargv.append(optstr)

    if logging.getLogger().level == logging.DEBUG:
        myargv.append("-d")
    elif logging.getLogger().level == logging.INFO:
        myargv.append("-v")

    if logging.getLogger().level in (logging.DEBUG, logging.INFO, logging.WARN):
        myargv.append("--traceback")

    if logfile:
        myargv.append("--logfile=%s" % logfile)

    return myargv

def get_options_from_ks(ksfile, parser):
    # readin the firstline of kickstart file for inline options, format as:
    # -*-mic2-options-*- --format=livecd --arch=arm -*-mic2-options-*-
    # the priority of different optinos source: cmdln_options >> inline_options
    # the extra inline args (left part from options) will be ignored
    first_line = None
    if ksfile and os.path.exists(ksfile):
        with open(ksfile) as f:
            first_line = f.readline()
    if first_line:
        m = re.match('^#\s*-\*-mic2-options-\*-\s+(.*)\s+-\*-mic2-options-\*-', first_line)
        if m:
            inline_argv = m.group(1).strip().split()
            if inline_argv:
                try:
                    (inline_opts, inline_args) = parser.parse_args(inline_argv)
                    return inline_opts
                except (optparse.BadOptionError, optparse.OptionValueError):
                    raise FatalError("Error: the options '%s' in the magic line of ks are invalid" %  m.group(1).strip())
    return None

def merge_options(options, options_plus):
    optlist = options.__dict__
    for key in optlist.keys():
        value = getattr(options, key)
        if value in (None, False, "none", ""):
            value = getattr(options_plus, key)
            setattr(options, key, value)

def get_all_parent_dirs(dir):
    dirs = []
    while True:
        parentdir = os.path.dirname(dir)
        if parentdir == "/":
            break
        dirs.append(parentdir)
        dir = parentdir
    return dirs

def order_bindmounts(bindmounts):
    final_bindmounts = ""
    dirs = bindmounts.split(";")
    if "" in dirs:
        dirs.remove("")
    if "/" in dirs:
        raise FatalError("Error: any one of cachedir, tmpdir, outdir, locale repo dir and logfile dir shouldn't be /")
    for i in range(len(dirs)):
        dirs[i] = dirs[i].strip()
        dirs[i] = dirs[i].replace("//", "/")
        if dirs[i][-1] == "/":
            dirs[i] = dirs[i][0:-1]
    dirs = list(set(dirs))
    i = 0
    swap = False
    while i < len(dirs):
        parentdirs = get_all_parent_dirs(dirs[i])
        for parentdir in parentdirs:
            if parentdir in dirs:
                j = dirs.index(parentdir)
                if j > i:
                    temp = dirs[i]
                    dirs[i] = dirs[j]
                    dirs[j] = temp
                    swap = True
        if swap:
            i = 0
            swap = False
        else:
            i = i + 1

    return ";".join(dirs) + ";"

def remove_bootstrap(bootstrap_root):
    if os.path.isdir(bootstrap_root):
        ret = mychroot.cleanup_mounts(bootstrap_root)
        if ret == 0:
            shutil.rmtree(bootstrap_root, ignore_errors = True)
        else:
            raise FatalError("Failed to rm %s, please ensure there isn't any process accessing some directories under %s" % (bootstrap_root, bootstrap_root))

def main():
    utils.misc.setlocale()
    global parser
    utils.misc.check_mic_installation(sys.argv)
    options = parse_options(sys.argv[1:])
    print "MIC2 version: %s" % version

    if os.geteuid () != 0:
        raise FatalError("You must run mic-image-creator as root")

    if options.siteconf and not os.path.isfile(options.siteconf):
        raise FatalError("Error: site config %s doesn't exist" % options.siteconf)

    if not options.ignore_siteconf:
        siteconf = utils.misc.read_siteconf(options.siteconf)
        logging.info(utils.misc.output_siteconf(siteconf))
    else:
        siteconf = None

    if not options.config and not options.ignore_siteconf:
        options.config = get_siteconf_opt(siteconf, "main", "config")
    options_from_ks = get_options_from_ks(options.config, parser)
    if options_from_ks:
        merge_options(options, options_from_ks)

    """ Get global config from site config, but command line option has high priority, it will override settings from site config """
    if not options.ignore_siteconf:
        get_options_from_siteconf(siteconf, options)

    if options.run_mode  == -1:
        if not os.path.exists("/etc/fedora-release") and not os.path.exists("/etc/meego-release"):
            options.run_mode = 1
        else:
            options.run_mode = 0
    if options.run_mode not in (0, 1):
        raise FatalError("Error: invalid run mode %d" % options.run_mode)

    # kickstart file isn't necessary any more
    if options.config and not os.path.isfile(options.config):
        raise FatalError("kickstart config '%s' does'not exist" % options.config)

    if options.record_pkgs is not None and options.record_pkgs not in ('name', 'content'):
        raise FatalError("Please specify what to be recorded: \"name\", \"content\"")

    if options.local_pkgs_path:
        options.local_pkgs_path = os.path.abspath(os.path.expanduser(options.local_pkgs_path))
        if not os.path.exists(options.local_pkgs_path):
            raise FatalError("Path %s doesn't exist" % options.local_pkgs_path)

    utils.misc.get_local_distro()

    distro_name = get_siteconf_opt(siteconf, "main", "distro_name")
    if not distro_name:
        distro_name = "MeeGo"
    if not options.prefix:
        options.prefix = distro_name.lower()

    if not os.path.exists("/etc/fedora-release") and not os.path.exists("/etc/meego-release"):
        """ For non-Fedora and non-MeeGo systems, bootstrap mode is mandatory """
        if options.run_mode == 0:
            raise FatalError("For non-Fedora and non-MeeGo systems, run mode must be 1")
    if imgcreate.is_meego_bootstrap("/"):
        """ No need to use bootstrap for MeeGo or bootstrap """
        options.run_mode = 0

    if options.run_mode == 1:
        """ If run_mode selected but no bootstrap is defined create env with defaults. """
        if not options.buildbootstrap and not options.bootstrap:
            options.bootstrap = "/var/cache/meego-bootstrap"
            if not os.path.exists(options.bootstrap):
                os.makedirs(options.bootstrap)
            if not imgcreate.is_meego_bootstrap(options.bootstrap) or options.rebuild_bootstrap:
                remove_bootstrap(options.bootstrap)
                options.buildbootstrap = True

    if options.rebuild_bootstrap:
        if options.bootstrap and os.path.isdir(options.bootstrap) and len(os.listdir(options.bootstrap)) != 0:
            remove_bootstrap(options.bootstrap)
        options.buildbootstrap = True

    if options.buildbootstrap:
        if not options.bootstrap:
            raise FatalError("Please use -b | --build-bootstrap or --rebuild-bootstrap with -B | --bootstrap together")
        elif not os.path.exists(options.bootstrap):
            try:
                os.makedirs(options.bootstrap)
            except OSError, (err, msg):
                raise FatalError(msg)
        elif os.path.isdir(options.bootstrap) and len(os.listdir(options.bootstrap)) != 0:
            raise FatalError("bootstrap must be an empty or non-existed directory.")

    if not options.buildbootstrap and options.bootstrap:
        if not os.path.isdir(options.bootstrap) or not utils.bootstrap.isbootstrap(options.bootstrap):
            raise FatalError("Please specify a workable bootstrap env")
    if options.bootstrap:
        print "Run mode: bootstrap"
    else:
        print "Run mode: legacy"

    if options.format not in (imgcreate.supported_formats + appcreate.supported_formats):
        raise FatalError("Image format '%s' isn't supported. Supported formats: %s " % (options.format, ", ".join(imgcreate.supported_formats + appcreate.supported_formats)))

    if options.package not in ("none", "tar", "tar.gz", "tar.bz2"):
        raise FatalError("Package format '%s' isn't supported" % options.package)

    if options.format == 'nand':
        print "WARNING: Image format 'nand' will be deprecated soon, and format 'mrstnand' is preferred."
        options.format = 'mrstnand'

    if not options.tmpdir:
        options.tmpdir = DEFAULT_TMPDIR

    options.tmpdir = os.path.abspath(os.path.expanduser(options.tmpdir))
    if not os.path.exists(options.tmpdir):
        os.makedirs(options.tmpdir)

    print "Using %s as tmpdir." % (options.tmpdir)

    if not options.cachedir:
        """ If no cache dir is found from configs lets create dir to users home as default. """
        options.cachedir = DEFAULT_CACHEDIR

    options.cachedir = os.path.abspath(os.path.expanduser(options.cachedir))
    if not os.path.exists(options.cachedir):
        os.makedirs(options.cachedir)

    print "Using %s as cache directory." % (options.cachedir)

    if not options.outdir:
        """ If no outdir is defined then we default to the current directory. """
        options.outdir = "."

    # Check if there is @ARCH@ values in .ks file and handle them accordingly.
    if options.config:
        original_ks_file = options.config
        fd = open(options.config, "r")
        kscont = fd.read()
        fd.close()

        # If we bootstrap we should not build ARM bootstrap for building
        # images so assign ia32 in that case as well. Next iteration will use
        # proper arm arch if needed.
        if options.bootstrap \
           or not options.arch \
           or options.arch in ("i386", "i486", "i586", "i686"):
            repo_arch = "ia32"
        else:
            repo_arch = options.arch

        kscont = kscont.replace("@ARCH@", repo_arch)

        if options.release and kscont.find("/@BUILD_ID@/") != -1:
            kscont = kscont.replace("@BUILD_ID@", options.release)

        base = os.path.basename(options.config)
        options.config = "%s/%s" %(options.tmpdir, base)
        fd = open(options.config, "w")
        fd.write(kscont)
        fd.close()
    else:
        original_ks_file = None


    releaseksbasename = None
    if options.release and options.run_mode == 0:
        options.record_pkgs = "name"
        if not options.suffix:
            options.suffix = options.release
        meegovert = ["handset", "netbook", "tablet", "ivi", "cetv", "core", "tv"]
        meegoarch = ["ia32", "armv5tel", "armv7l", "armv7hl", "armv7nhl"]
        base = os.path.basename(options.config)
        ksname = base[:base.rfind(".")]
        ksnamec = ksname.split("-")

        """ Accept also the image name prefix in kickstart filename. """
        if ksnamec[0] == options.prefix:
            ksnamec.pop(0)

        """ Ignore release id in kickstart """
        if re.match("\d+\.\d+.\d+.*$", ksnamec[-1]):
            ksnamec.pop(-1)

        """ There might be more than 3 parts in the ksnamec because of the release numbering. """
        if len(ksnamec) < 2 or ksnamec[0] not in meegovert or ksnamec[1] not in meegoarch:
            raise FatalError("Configuration file %s does not follow naming conventions: $VERTICAL-$ARCH-$VARIANT or %s-$VERTICAL-$ARCH-$VARIANT" %(options.config,options.prefix))

        """ Use the ksnamec array here, as we do not want to keep the old release name that might come after $VARIANT in kickstart filename. """
        releaseksbasename = "-".join(ksnamec)
        options.outdir = "%s/%s/images/%s-%s" %(options.outdir, options.release, options.prefix, releaseksbasename)

    options.outdir = os.path.abspath(os.path.expanduser(options.outdir))

    if not os.path.exists(options.outdir):
        imgcreate.fs.makedirs(options.outdir)

    print "Using %s as output directory." % (options.outdir)

    """ proxy settings from site config override proxy settings from environment variables """
    utils.misc.set_proxies(get_siteconf_opt(siteconf,"main","proxy"), \
                               get_siteconf_opt(siteconf,"main","no_proxy"))

    """ Get repos from site config """
    site_repos = utils.misc.get_repostrs_from_siteconf(siteconf)

    """ Give a temporary name for repos without name """
    if options.addrepos:
        for i in range(len(options.addrepos)):
            if options.addrepos[i].split(":")[0] in ("http", "https", "ftp", "ftps"):
                options.addrepos[i] = "name:%s,baseurl:%s" % (utils.misc.get_temp_reponame(options.addrepos[i]), options.addrepos[i])
    else:
       options.addrepos = []

    """ Must call get_metadata_from_repos firstly to get repo metadata """
    ks_repos = []
    if options.config:
        try:
            ks = imgcreate.read_kickstart(options.config)
            ks_repos = utils.misc.get_repostrs_from_ks(ks)
        except imgcreate.CreatorError, e:
            raise FatalError("unable to load kickstart file '%s' : %s" % (options.config, e))

    try:
        repometadata = utils.misc.get_metadata_from_repos(options.addrepos + site_repos + ks_repos, options.cachedir)
        if not repometadata:
            raise FatalError("No repositories found")
        release_no = utils.misc.get_release_no(repometadata, distro_name.lower())
        target_archlist = utils.misc.get_arch(repometadata)
    except (OSError,imgcreate.CreatorError), e:
        logging.exception(e)
        raise FatalError("failed to create image : %s" % e)

    print "Available target architectures in repositories: %s" % target_archlist
    print "\n%s release %s\n" % (distro_name, release_no)

    if not options.config:
        try:
            ksfiles = utils.misc.get_kickstarts_from_repos(repometadata)
        except imgcreate.CreatorError, e:
            logging.exception(e)
            raise FatalError("failed to create image : %s" % e)
        if not ksfiles:
            """ Didn't find kickstart files from repo, so try to get it from command line option """
            raise FatalError("Your repos do not provide any kickstart files, please provide a kickstart file using -c or --config option.")
        else:
            if not options.default_ks:
                options.default_ks = get_siteconf_opt(siteconf, "main", "default_ks")
            if options.default_ks:
                for ksfile in ksfiles:
                    if options.default_ks == os.path.basename(ksfile["filename"]):
                        options.config = ksfile["filename"]
                        break
                if not options.config:
                    raise FatalError("Failed to find default kickstart file %s" % options.default_ks)
            else:
                options.config = utils.misc.select_ks(ksfiles)

    try:
        ks = imgcreate.read_kickstart(options.config)
    except imgcreate.CreatorError, e:
        logging.exception(e)
        raise FatalError("unable to load kickstart file '%s' : %s" % (options.config, e))

    """ Check image format for arm """
    errstr = "livecd or liveusb image can't support arm, please use other image formats"
    imgfmts_not_for_arm = ['livecd','liveusb']
    if options.arch and options.arch.startswith("arm") and options.format in imgfmts_not_for_arm:
        raise FatalError(errstr)

    """ If a user needs to use btrfs or creates ARM image, selinux must be disabled by her/him before running image creator"""
    if os.path.exists("/etc/fedora-release"):
        dev_null = os.open("/dev/null",os.O_WRONLY)
        pobj = subprocess.Popen(["/usr/sbin/getenforce"],stdout=subprocess.PIPE,stderr=dev_null)
        selinux_status = pobj.communicate()[0]
        os.close(dev_null)
        selinux_status = selinux_status.rstrip('\n')
        if  options.arch and options.arch.startswith("arm") and selinux_status == "Enforcing":
                raise FatalError("Can't create arm image if selinux is enabled, please disbale it before running: $ sudo setenforce 0")

        use_btrfs = False
        parts = ks.handler.partition.partitions
        for i in range(len(parts)):
            if parts[i].fstype == "btrfs":
                use_btrfs = True
        if use_btrfs and selinux_status == "Enforcing":
            raise FatalError("Fail to create btrfs filesystem if selinux is enabled, please disbale it before running: $ sudo setenforce 0")

    for siterepo in site_repos:
        try:
           imgcreate.kickstart.add_repo(ks, utils.misc.get_repostr(siterepo, siteconf))
        except ValueError, msg:
           raise FatalError("Please provide valid repo string")

    """ Get repostrs from kickstart """
    ks_repos = utils.misc.get_repostrs_from_ks(ks)
    imgcreate.kickstart.remove_all_repos(ks) # Need to automatically set proxy for repos or remap repos
    for ksrepo in ks_repos:
        try:
           imgcreate.kickstart.add_repo(ks, utils.misc.get_repostr(ksrepo, siteconf))
        except ValueError, msg:
           raise FatalError("Please provide valid repo string")

    """ Set additional repos """
    if options.addrepos:
        for addrepo in options.addrepos:
            try:
                imgcreate.kickstart.add_repo(ks, utils.misc.get_repostr(addrepo, siteconf))
            except ValueError, msg:
                raise FatalError("Please provide valid repo string")

    """ Remove duplicate repos """
    imgcreate.kickstart.remove_duplicate_repos(ks)

    if options.release and not options.suffix:
        options.suffix = options.release
    if not options.suffix:
        options.suffix = "%s.%s" %(release_no, time.strftime("%Y%m%d.%H%M"))

    ksname = options.config
    """ With release we have stripped old release tags out of the name. """
    if releaseksbasename:
      ksname = releaseksbasename

    name = imgcreate.build_name(ksname, "%s-" % options.prefix, suffix=options.suffix)
    print "Image name: %s" % (name)

    bootstrap_target_arch = None
    if options.arch:
        if options.bootstrap:
            # We do not want to do ARM bootstrap env even if building ARM images.
            # So change to i686 for now.
            bootstrap_target_arch = options.arch
            options.arch = "i686"
            print "Changing architecture from '%s' to '%s' for the bootstrap creation." % (bootstrap_target_arch,options.arch)
        elif options.arch.strip() in ("i386", "i486", "i586", "i686"):
            options.arch = "i686"
        arch_set = False
        for target_arch in target_archlist:
            if target_arch.startswith(options.arch.strip()):
                options.arch = target_arch
                print "Target arch: %s" % options.arch
                arch_set = True
                break

        if not arch_set and not bootstrap_target_arch:
            raise FatalError("Given arch '%s' is not found from the repositories." % options.arch.strip())

    need_convert = False
    if options.bootstrap and options.format in ("vmdk", "vdi"):
        utils.bootstrap.check_depends(options.format)
        need_convert = True
        imgfmt = options.format
        options.format = "raw"
        pkgfmt = options.package
        options.package = "none"

    if options.buildbootstrap:
        try:
            utils.bootstrap.build_bootstrap(ks.handler.repo.repoList, 
                                                options.mainrepo,
                                                options.cachedir,
                                                options.bootstrap,
                                                bootstrap_arch = options.arch,
                                                bootstrap_target_arch = bootstrap_target_arch)
        except imgcreate.CreatorError, e:
            logging.exception(e)
            raise FatalError("Failed to create bootstrap : %s" % e)

    if options.bootstrap:
        if bootstrap_target_arch:
            # Changing back to the target arch if set.
            options.arch = bootstrap_target_arch

        utils.bootstrap.check_mic(options.bootstrap)
        """ Copy the .ks file to the bootstrap """
        ks_in_bootstrap = options.bootstrap + "/root/" + os.path.basename(options.config)
        shutil.copy(original_ks_file, ks_in_bootstrap)
        """ Change the config to point to the .ks file inside the bootstrap. """
        options.config = "/root/" + os.path.basename(options.config)

        logfile = utils.debug.get_logfile()
        argv = rebuild_argv(sys.argv, options, logfile)
        mypipe = tempfile.mktemp(".mic2pipe", ".mic2-", options.outdir)
        argv.append("--pipe=%s" % mypipe)
        if need_convert:
            image_info = tempfile.mktemp(".mic2info", ".mic2-", options.outdir)
            argv.append("--image-info=%s" % image_info)
        print argv
        bindmounts = ""
        for repo in ks.handler.repo.repoList:
            if repo.baseurl.startswith("file:///"):
                repodir = repo.baseurl.replace("file://", "")
                if bindmounts.find(repodir) < 0:
                    bindmounts += "%s;" % repodir
        bindmounts += "%s;%s;" % (options.outdir, options.tmpdir)
        if options.cachedir:
            bindmounts = bindmounts + "%s;" % options.cachedir
        if logfile:
            logdir = os.path.dirname(logfile)
            bindmounts += "%s;" % logdir
        if options.local_pkgs_path:
            bindmounts += "%s;" % options.local_pkgs_path
        bindmounts = order_bindmounts(bindmounts)
        """ Preload the required modules before entering bootstrap """
        imgcreate.fs.load_module("btrfs")
        imgcreate.fs.load_module("dm_snapshot")
        run_in_bootstrap(options.bootstrap, argv, bindmounts, bootstrap_target_arch)
        # For the case with release option, destdir is changed, so get it
        for line in open(mypipe).xreadlines():
            if line.find("destdir=") != -1:
                options.outdir = line[line.find("destdir=") + 8:-1]
            if line.find("package=") != -1:
                pkgfmt = line[line.find("package=") + 8:-1]
        os.unlink(mypipe)
        if need_convert:
            imgfile = utils.bootstrap.get_imgfile(image_info)
            srcdir = os.path.dirname(imgfile)
            dstdir = srcdir[0:-3] + imgfmt
            shutil.move(srcdir, dstdir)
            imgfile = dstdir + "/" + os.path.basename(imgfile)
            outimages = utils.bootstrap.convert_to(imgfile, imgfmt)
            dst = utils.bootstrap.package_image(outimages, imgfmt, options.outdir, pkgfmt)
            if options.release:
                outimages = imgcreate.create_release(original_ks_file, options.outdir, name, outimages, options.release)
            if dst:
                outimages.append(dst)
            utils.bootstrap.print_imginfo(outimages)
            print "Finished."
        """ Remove the .ks file that is inside bootstrap, because the bootstrap mic thinks it is the original. """
        os.unlink(ks_in_bootstrap)
        return 0


    if options.format == "fs":
        creator = imgcreate.FsImageCreator(ks, name)
    elif options.format in ("livecd", "liveusb"):
        if options.volumeid:
            fs_label = options.volumeid
            if len(fs_label) > imgcreate.FSLABEL_MAXLEN:
                fs_label = fs_label[:imgcreate.FSLABEL_MAXLEN]
        else:
            fs_label = imgcreate.build_name(options.config,
                                            "%s-" % options.prefix,
                                            maxlen = imgcreate.FSLABEL_MAXLEN,
                                            suffix = "%s-%s" %(os.uname()[4], time.strftime("%Y%m%d%H%M")))

        logging.info("Using label '%s' and name '%s'" % (fs_label, name))
        try:
            if options.format == "liveusb" and not options.interactive:
                creator = imgcreate.LiveUSBImageCreator(ks, name, fs_label)
                creator.fstype = options.fstype
                if creator.fstype != "vfat" and creator.fstype != "ext3":
                    raise FatalError("file system type for live USB image  must be vfat or ext3")
            else:
                creator = imgcreate.LiveImageCreator(ks, name, fs_label)
            if options.format == "liveusb":
                creator.overlaysizemb = options.overlay_size_mb
                if creator.overlaysizemb < 0:
                    raise FatalError("Overlay size must be greater than 0")
        except imgcreate.CreatorError, e:
            logging.exception(e)
            raise FatalError("failed to create image : %s" % e)
        finally:
            creator.cleanup()

        creator.skip_compression = options.skip_compression
        creator.skip_minimize = options.skip_minimize
    elif options.format == "loop":
        creator = imgcreate.LoopImageCreator(ks, name)
    elif options.format == "mrstnand":
        creator = appcreate.NandImageCreator(ks, name,
                                initrd_url = options.initrd_url,
                                initrd_path = options.initrd_path,
                                kernel_url = options.kernel_url,
                                kernel_path = options.kernel_path,
                                kernel_rpm_url = options.kernel_rpm_url,
                                kernel_rpm_path = options.kernel_rpm_path,
                                bootimg_only = options.bootimg_only)
    elif options.format == "ubi":
        creator = imgcreate.UbiImageCreator(ks, name)
    elif options.format == "jffs2":
        creator = imgcreate.Jffs2ImageCreator(ks, name)
    elif options.format in ("raw", "vmdk", "vdi"):
        creator = appcreate.ApplianceImageCreator(ks, name, options.format, 512, 1)
    else:
        raise FatalError("You have to specify an image format")

    if options.compress_disk_image == "none":
        """ If user does not want compression for release for example """
        options.compress_disk_image = None
    elif not options.compress_disk_image and options.release:
        """ For livecd and liveusb image, unnecessary to compress """
        if options.format in ("livecd", "liveusb"):
            options.compress_disk_image = None
        else:
            """ If compression is not set then for release we set the default to bz2 """
            options.compress_disk_image = "bz2"

    if options.compress_disk_image:
        creator.compress_disk_image(options.compress_disk_image)

    creator.tmpdir = options.tmpdir
    creator._alt_initrd_name = options.alt_initrd_name
    creator._recording_pkgs = options.record_pkgs
    creator._include_src = options.include_src
    creator._local_pkgs_path = options.local_pkgs_path
    creator._genchecksum = options.genchecksum
    creator.distro_name = distro_name
    creator.image_format = options.format

    if options.arch:
        creator.set_target_arch(options.arch)
    if options.pkgmgr:
        creator.set_pkg_manager(options.pkgmgr)

    """
    zypp can't recognize the group in comps, yum can't recognize the group in
    patterns, either, resolve_groups will replace the related groups with their
    package list if the given package manager can't recognize them
    """
    imgcreate.kickstart.resolve_groups(creator, repometadata, options.use_comps)

    try:
        creator.check_depend_tools()
        creator.mount(None, options.cachedir)

        creator.install()

        ''' Download the source packages '''
        if options.include_src and options.format == "fs":
            installed_pkgs =  creator.get_installed_packages()
            print '--------------------------------------------------'
            print 'Generating the image with source rpms included, The number of source packages is %d.' %(len(installed_pkgs))
            if not utils.misc.SrcpkgsDownload(installed_pkgs, repometadata, creator._instroot, options.cachedir):
                print "Source packages can't be downloaded"

        creator.configure(repometadata)
        if options.save_kernel:
            creator.save_kernel(options.outdir)
        creator.launch_shell(options.give_shell)
        creator.unmount()
        creator.package(options.outdir)
        outimage = creator.outimage
        if not options.image_info:
            if options.release:
                creator.release_output(original_ks_file, options.outdir, name, options.release)
            creator.package_output(options.format, options.outdir, options.package)
            creator.print_outimage_info()
            outimage = creator.outimage
        else:
            utils.bootstrap.save_imginfo(outimage, options.image_info)
        if options.pipe:
            fd = open(options.pipe, "w")
            print "Pipe outdir: %s" % options.outdir
            fd.write("destdir=%s\n" % options.outdir)
            if options.release and options.package:
                fd.write("package=%s\n" % options.package)
                print "Pipe package: %s" % options.package
            fd.close()
    except imgcreate.CreatorError, e:
        logging.exception(e)
        raise FatalError("failed to create image : %s" % e)
    finally:
        creator.cleanup()

    if options.format == "liveusb" and options.interactive:
        mypath = os.path.abspath(sys.argv[0])
        mypath = os.path.dirname(mypath)
        isotodisk = mypath + "/mic-livecd-iso-to-disk"
        if not os.path.isfile(isotodisk):
            raise FatalError("Error mic-livecd-iso-to-disk doesn't exist")
        iso = [x for x in outimage if x.endswith(".iso")]
        args = [isotodisk, "--reset-mbr", "--overlay-size-mb", "%d" % creator.overlaysizemb, iso[0]]
        ret = subprocess.call(args)
        if ret != 0:
            raise FatalError("failed to create Live USB : '%s' exited with error (%d)" % (string.join(args, " "), ret))

    if not options.image_info:
            print "Finished."

    return 0


COLOR_BLACK = "\033[00m"
COLOR_RED =   "\033[1;31m"
if __name__ == "__main__":
    try:
        ret = main()
        utils.terminate_log_thread()
        sys.exit(ret)
    except (imgcreate.CreatorError, FatalError, utils.bootstrap.BootstrapError), e:
        print >> sys.stderr, "\n%sError: %s%s\n" % (COLOR_RED, e, COLOR_BLACK)
        logging.exception(e)
        utils.terminate_log_thread()
        sys.exit(1)