dh-python 2.20151103ubuntu1 / usr / share / dh-python / dhpython / interpreter.py

# Copyright © 2012-2013 Piotr Ożarowski <piotr@debian.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import logging
import os
import re
from os.path import join, split
from dhpython import INTERPRETER_DIR_TPLS, PUBLIC_DIR_RE, OLD_SITE_DIRS

SHEBANG_RE = re.compile(r'''
    (?:\#!\s*){0,1}  # shebang prefix
    (?P<path>
        .*?/bin/.*?)?
    (?P<name>
        python|pypy)
    (?P<version>
        \d[\.\d]*)?
    (?P<debug>
        -dbg)?
    (?P<options>.*)
    ''', re.VERBOSE)
EXTFILE_RE = re.compile(r'''
    (?P<name>.*?)
    (?:\.
        (?P<stableabi>abi\d+)
     |(?:\.
        (?P<soabi>
            (?P<impl>cpython|pypy)
            -
            (?P<ver>\d{2})
            (?P<flags>[a-z]*)
        )?
        (?:
            (?:(?<!\.)-)?  # minus sign only if soabi is defined
            (?P<multiarch>[^/]*?)
        )?
    ))?
    (?P<debug>_d)?
    \.so$''', re.VERBOSE)
log = logging.getLogger('dhpython')


class Interpreter:
    """
    :attr path: /usr/bin/ in most cases
    :attr name: pypy or python (even for python3 and python-dbg) or empty string
    :attr version: interpreter's version
    :attr debug: -dbg version of the interpreter
    :attr impl: implementation (cpytho2, cpython3 or pypy)
    :attr options: options parsed from shebang
    :type path: str
    :type name: str
    :type version: Version or None
    :type debug: bool
    :type impl: str
    :type options: tuple
    """
    path = '/usr/bin/'
    name = 'python'
    version = None
    debug = False
    impl = ''
    options = ()
    _cache = {}

    def __init__(self, value=None, path=None, name=None, version=None,
                 debug=None, impl=None, options=None):
        params = locals()
        del params['self']
        del params['value']

        if isinstance(value, Interpreter):
            for key in params.keys():
                if params[key] is None:
                    params[key] = getattr(value, key)
        elif value:
            if value.replace('.', '').isdigit() and not version:
                # version string
                params['version'] = Version(value)
            else:
                # shebang or other string
                for key, val in self.parse(value).items():
                    # prefer values passed to constructor over shebang ones:
                    if params[key] is None:
                        params[key] = val

        for key, val in params.items():
            if val is not None:
                setattr(self, key, val)
            elif key == 'version':
                setattr(self, key, val)

    def __setattr__(self, name, value):
        if name == 'name':
            if value not in ('python', 'pypy', ''):
                raise ValueError("interpreter not supported: %s" % value)
            if value == 'python':
                if self.version:
                    if self.version.major == 3:
                        self.__dict__['impl'] = 'cpython3'
                    else:
                        self.__dict__['impl'] = 'cpython2'
            elif value == 'pypy':
                self.__dict__['impl'] = 'pypy'
        elif name == 'version' and value is not None:
            value = Version(value)
            if not self.impl and self.name == 'python':
                if value.major == 3:
                    self.impl = 'cpython3'
                else:
                    self.impl = 'cpython2'
        if name in ('path', 'name', 'impl', 'options') and value is None:
            pass
        elif name == 'debug':
            self.__dict__[name] = bool(value)
        else:
            self.__dict__[name] = value

    def __repr__(self):
        result = self.path
        if not result.endswith('/'):
            result += '/'
        result += self._vstr(self.version)
        if self.options:
            result += ' ' + ' '.join(self.options)
        return result

    def __str__(self):
        return self._vstr(self.version)

    def _vstr(self, version=None, consider_default_ver=False):
        if self.impl == 'pypy':
            # TODO: will Debian support more than one PyPy version?
            return self.name
        version = version or self.version or ''
        if consider_default_ver and (not version or version == self.default_version):
            version = '3' if self.impl == 'cpython3' else ''
        elif isinstance(version, Version) and version == Version(major=2):
            version = ''  # do not promote /usr/bin/python2
        if self.debug:
            return 'python{}-dbg'.format(version)
        return self.name + str(version)

    def binary(self, version=None):
        return '{}{}'.format(self.path, self._vstr(version))

    @property
    def binary_dv(self):
        """Like binary(), but returns path to default intepreter symlink
        if version matches default one for given implementation.
        """
        return '{}{}'.format(self.path, self._vstr(consider_default_ver=True))

    @property
    def default_version(self):
        if self.impl:
            return default(self.impl)

    @staticmethod
    def parse(shebang):
        """Return dict with parsed shebang

        >>> sorted(Interpreter.parse('/usr/bin/python3.2-dbg').items())
        [('debug', '-dbg'), ('name', 'python'), ('options', ()), ('path', '/usr/bin/'), ('version', '3.2')]
        >>> sorted(Interpreter.parse('#! /usr/bin/python3.2').items())
        [('debug', None), ('name', 'python'), ('options', ()), ('path', '/usr/bin/'), ('version', '3.2')]
        >>> sorted(Interpreter.parse('/usr/bin/python3.2-dbg --foo --bar').items())
        [('debug', '-dbg'), ('name', 'python'), ('options', ('--foo', '--bar')),\
 ('path', '/usr/bin/'), ('version', '3.2')]
        """
        result = SHEBANG_RE.search(shebang)
        if not result:
            return {}
        result = result.groupdict()
        if 'options' in result:
            # TODO: do we need "--key value" here?
            result['options'] = tuple(result['options'].split())
        if result['name'] == 'python' and result['version'] is None:
            result['version'] = '2'
        return result

    @classmethod
    def from_file(cls, fpath):
        """Read file's shebang and parse it."""
        interpreter = Interpreter()
        with open(fpath, 'rb') as fp:
            data = fp.read(96)
            if b"\0" in data:
                raise ValueError('cannot parse binary file')
        # make sure only first line is checkeed
        data = str(data, 'utf-8').split('\n')[0]
        if not data.startswith('#!'):
            raise ValueError("doesn't look like a shebang: %s" % data)

        parsed = cls.parse(data)
        if not parsed:
            raise ValueError("doesn't look like a shebang: %s" % data)
        for key, val in parsed.items():
            setattr(interpreter, key, val)
        return interpreter

    def sitedir(self, package=None, version=None, gdb=False):
        """Return path to site-packages directory.

        Note that returned path is not the final location of .py files

        >>> i = Interpreter('python')
        >>> i.sitedir(version='3.1')
        '/usr/lib/python3/dist-packages/'
        >>> i.sitedir(version='2.5')
        '/usr/lib/python2.5/site-packages/'
        >>> i.sitedir(version=Version('2.7'))
        '/usr/lib/python2.7/dist-packages/'
        >>> i.sitedir(version='3.1', gdb=True, package='python3-foo')
        'debian/python3-foo/usr/lib/debug/usr/lib/python3/dist-packages/'
        >>> i.sitedir(version=Version('3.2'))
        '/usr/lib/python3/dist-packages/'
        """
        try:
            version = Version(version or self.version)
        except Exception as err:
            raise ValueError("cannot find valid version: %s" % err)
        if self.impl == 'pypy':
            path = '/usr/lib/pypy/dist-packages/'
        elif version << Version('2.6'):
            path = "/usr/lib/python%s/site-packages/" % version
        elif version << Version('3.0'):
            path = "/usr/lib/python%s/dist-packages/" % version
        else:
            path = '/usr/lib/python3/dist-packages/'

        if gdb:
            path = "/usr/lib/debug%s" % path
        if package:
            path = "debian/%s%s" % (package, path)

        return path

    def old_sitedirs(self, package=None, version=None, gdb=False):
        """Return deprecated paths to site-packages directories."""
        try:
            version = Version(version or self.version)
        except Exception as err:
            raise ValueError("cannot find valid version: %s" % err)
        result = []
        for item in OLD_SITE_DIRS.get(self.impl, []):
            if isinstance(item, str):
                result.append(item.format(version))
            else:
                res = item(version)
                if res is not None:
                    result.append(res)

        if gdb:
            result = ['/usr/lib/debug{}'.format(i) for i in result]
            if self.impl.startswith('cpython'):
                result.append('/usr/lib/debug/usr/lib/pyshared/python{}'.format(version))
        if package:
            result = ['debian/{}{}'.format(package, i) for i in result]

        return result

    def parse_public_dir(self, path):
        """Return version assigned to site-packages path
        or True is it's unversioned public dir."""
        match = PUBLIC_DIR_RE[self.impl].match(path)
        if match:
            vers = match.groups(0)
            if vers and vers[0]:
                return Version(vers)
            return True

    def should_ignore(self, path):
        """Return True if path is used by another interpreter implementation."""
        cache_key = 'should_ignore_{}'.format(self.impl)
        if cache_key not in self.__class__._cache:
            expr = [v for k, v in INTERPRETER_DIR_TPLS.items() if k != self.impl]
            regexp = re.compile('|'.join('({})'.format(i) for i in expr))
            self.__class__._cache[cache_key] = regexp
        else:
            regexp = self.__class__._cache[cache_key]
        return regexp.search(path)

    def cache_file(self, fpath, version=None):
        """Given path to a .py file, return path to its .pyc/.pyo file.

        This function is inspired by Python 3.2's imp.cache_from_source.

        :param fpath: path to file name
        :param version: Python version

        >>> i = Interpreter('python')
        >>> i.cache_file('foo.py', Version('3.1'))
        'foo.pyc'
        >>> i.cache_file('bar/foo.py', '3.4')
        'bar/__pycache__/foo.cpython-34.pyc'
        """
        version = Version(version or self.version)
        last_char = 'o' if '-O' in self.options else 'c'
        if version <= Version('3.1'):
            return fpath + last_char

        fdir, fname = split(fpath)
        if not fname.endswith('.py'):
            fname += '.py'
        return join(fdir, '__pycache__', "%s.%s.py%s" %
                    (fname[:-3], self.magic_tag(version), last_char))

    def magic_number(self, version=None):
        """Return magic number."""
        version = Version(version or self.version)
        if self.impl == 'cpython2':
            return ''
        result = self._execute('import imp; print(imp.get_magic())', version)
        return eval(result)

    def magic_tag(self, version=None):
        """Return Python magic tag (used in __pycache__ dir to tag files).

        >>> i = Interpreter('python')
        >>> i.magic_tag(version='3.4')
        'cpython-34'
        """
        version = Version(version or self.version)
        if self.impl.startswith('cpython') and version << Version('3.2'):
            return ''
        return self._execute('import imp; print(imp.get_tag())', version)

    def multiarch(self, version=None):
        """Return multiarch tag."""
        version = Version(version or self.version)
        try:
            soabi, multiarch = self._get_config(version)[:2]
        except Exception:
            log.debug('cannot get multiarch', exc_info=True)
            # interpreter without multiarch support
            return ''
        return multiarch

    def stableabi(self, version=None):
        version = Version(version or self.version)
        # stable ABI was introduced in Python 3.3
        if self.impl == 'cpython3' and version >> Version('3.2'):
            return 'abi{}'.format(version.major)

    def soabi(self, version=None):
        """Return SOABI flag (used to in .so files)."""
        version = Version(version or self.version)
        # NOTE: it's not the same as magic_tag
        try:
            soabi, multiarch = self._get_config(version)[:2]
        except Exception:
            log.debug('cannot get soabi', exc_info=True)
            # interpreter without soabi support
            return ''
        return soabi

    @property
    def include_dir(self):
        """Return INCLUDE_DIR path.

        >>> Interpreter('python2.7').include_dir
        '/usr/include/python2.7'
        >>> Interpreter('python3.4-dbg').include_dir
        '/usr/include/python3.4dm'
        """
        if self.impl == 'pypy':
            return '/usr/lib/pypy/include'
        try:
            result = self._get_config()[2]
            if result:
                return result
        except Exception:
            result = ''
            log.debug('cannot get include path', exc_info=True)
        result = '/usr/include/{}'.format(self.name)
        version = self.version
        if self.debug:
            if version << '3.3':
                result += '_d'
            else:
                result += 'dm'
        else:
            if version >> '3.2':
                result += 'm'
            elif version == '3.2':
                result += 'mu'
        return result

    @property
    def library_file(self):
        """Return libfoo.so file path."""
        if self.impl == 'pypy':
            return ''
        libpl, ldlibrary = self._get_config()[3:5]
        if ldlibrary.endswith('.a'):
            # python3.1-dbg, python3.2, python3.2-dbg returned static lib
            ldlibrary = ldlibrary.replace('.a', '.so')
        if libpl and ldlibrary:
            return join(libpl, ldlibrary)
        raise Exception('cannot find library file for {}'.format(self))

    def check_extname(self, fname, version=None):
        """Return extension file name if file can be renamed."""
        if not version and not self.version:
            return

        version = Version(version or self.version)

        if '/' in fname:
            fdir, fname = fname.rsplit('/', 1)  # in case full path was passed
        else:
            fdir = ''

        info = EXTFILE_RE.search(fname)
        if not info:
            return
        info = info.groupdict()
        if info['ver'] and (not version or version.minor is None):
            # get version from soabi if version is not set of only major
            # version number is set
            version = Version("%s.%s" % (info['ver'][0], info['ver'][1]))

        if info['stableabi']:
            # files with stable ABI in name don't need changes
            return
        if info['debug'] and self.debug is False:
            # do not change Python 2.X extensions already marked as debug
            # (the other way arround is acceptable)
            return
        if info['soabi'] and info['multiarch']:
            # already tagged, nothing we can do here
            return

        try:
            soabi, multiarch = self._get_config(version)[:2]
        except Exception:
            log.debug('cannot get soabi/multiarch', exc_info=True)
            return

        if info['soabi'] and soabi and info['soabi'] != soabi:
            return

        tmp_soabi = info['soabi'] or soabi
        tmp_multiarch = info['multiarch'] or multiarch

        result = info['name']
        if result.endswith('module') and (self.impl == 'cpython3' and version >> '3.2'
                                          or self.impl == 'cpython2' and version == '2.7'):
            result = result[:-6]

        if tmp_soabi:
            result = "{}.{}".format(result, tmp_soabi)
            if tmp_multiarch and not (self.impl == 'cpython3' and version << '3.3') and tmp_multiarch not in soabi:
                result = "{}-{}".format(result, tmp_multiarch)
        elif self.impl == 'cpython2' and version == '2.7' and tmp_multiarch:
            result = "{}.{}".format(result, tmp_multiarch)

        if self.debug and self.impl == 'cpython2':
            result += '_d'
        result += '.so'
        if fname == result:
            return
        return join(fdir, result)

    def suggest_pkg_name(self, name):
        """Suggest binary package name with for given library name

        >>> Interpreter('python3.1').suggest_pkg_name('foo')
        'python3-foo'
        >>> Interpreter('python3.4').suggest_pkg_name('foo')
        'python3-foo'
        >>> Interpreter('python2.7-dbg').suggest_pkg_name('bar')
        'python-bar-dbg'
        """
        if self.impl == 'pypy':
            return 'pypy-{}'.format(name)
        version = '3' if self.impl == 'cpython3' else ''
        result = 'python{}-{}'.format(version, name)
        if self.debug:
            result += '-dbg'
        return result

    def _get_config(self, version=None):
        version = Version(version or self.version)
        # sysconfig module is available since Python 3.2
        # (also backported to Python 2.7)
        if self.impl == 'pypy' or self.impl.startswith('cpython') and (
                version >> '2.6' and version << '3'
                or version >> '3.1' or version == '3'):
            cmd = 'import sysconfig as s;'
        else:
            cmd = 'from distutils import sysconfig as s;'
        cmd += 'print("__SEP__".join(i or "" ' \
               'for i in s.get_config_vars('\
               '"SOABI", "MULTIARCH", "INCLUDEPY", "LIBPL", "LDLIBRARY")))'
        conf_vars = self._execute(cmd, version).split('__SEP__')
        if conf_vars[1] in conf_vars[0]:
            # Python >= 3.5 includes MILTIARCH in SOABI
            conf_vars[0] = conf_vars[0].replace("-%s" % conf_vars[1], '')
        try:
            conf_vars[1] = os.environ['DEB_HOST_MULTIARCH']
        except KeyError:
            pass
        return conf_vars

    def _execute(self, command, version=None, cache=True):
        version = Version(version or self.version)
        command = "{} -c '{}'".format(self._vstr(version), command.replace("'", "\'"))
        if cache and command in self.__class__._cache:
            return self.__class__._cache[command]

        output = execute(command)
        if output['returncode'] != 0:
            log.debug(output['stderr'])
            raise Exception('{} failed with status code {}'.format(command, output['returncode']))

        result = output['stdout'].splitlines()

        if len(result) == 1:
            result = result[0]

        if cache:
            self.__class__._cache[command] = result

        return result

# due to circular imports issue
from dhpython.tools import execute
from dhpython.version import Version, default