pybik 1.1.1-1 / usr / lib / pybik / pybiktest / testrunner.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-

#  Copyright © 2013-2014  B. Clausius <barcc@gmx.de>
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program.  If not, see <http://www.gnu.org/licenses/>.



import sys, os
import random
from collections import namedtuple, OrderedDict
import tempfile

from PyQt4.QtCore import Qt, QTimer
from PyQt4.QtGui import QAction
from PyQt4.QtTest import QTest

from . import utils
from pybiklib.settings import settings


class Logger (object):
    logfilename = 'pybiktest/test.log'
    
    def __init__(self):
        self.lineno = None
        self.result = None
        self.logfile = None
        
    def open(self, log):
        if not log:
            self.log('Logfile disabled\n')
            return
        try:
            self.logfile = open(self.logfilename, 'wt', encoding='utf-8')
            self.logf('Use logfile: {}\n', self.logfilename)
        except IOError as e:
            self.logfile = None
            self.log('Logfile disabled:', e)
            
    def close(self):
        if self.logfile is not None:
            self.logfile.close()
            self.logfile = None
            
    def log(self, *args, **kwargs):
        if self.lineno is not None:
            args = ('line {}:'.format(self.lineno),) + args
        print(*args, **kwargs)
        if self.logfile is not None:
            kwargs['file'] = self.logfile
            print(*args, **kwargs)
            
    def logf(self, format, *args, **kwargs):
        self.log(format.format(*args), **kwargs)
        
    def error(self, message, *args, **kwargs):
        kwargs['file'] = sys.stderr
        self.log(message, *args, **kwargs)
        message = message.rstrip(':')
        if message not in self.result:
            self.result[message] = 1
        else:
            self.result[message] += 1
        
    def errorf(self, message, format, *args, **kwargs):
        self.error(message, format.format(*args), **kwargs)
        
logger = Logger()
log = logger.log
logf = logger.logf
log_error = logger.error
logf_error = logger.errorf


class NamedObject (object):     # pylint: disable=R0903
    def __init__(self, name):
        self.name = name
    def __repr__(self):
        return self.name
matchall = NamedObject('matchall')
nomatch = NamedObject('nomatch')


def mkState(fields):
    _State = namedtuple('_State', fields)
    class State(_State):   # pylint: disable=W0232
        __slots__ = ()
        def tostr(self, other=None):
            def field_tostr(i):
                if self[i] is matchall:
                    return ''
                elif other is None or self[i] != other[i]:
                    return '{}={!r}, '.format(self.fields[i], self[i])
                else:
                    return ' ' * (len(self.fields[i]) + len(repr(self[i])) + 3)
            return ''.join([field_tostr(i) for i in range(len(self))]).rstrip(', ')
            
        fields = _State._fields
        asdict = _State._asdict
        replace = _State._replace
    State.default = State(**{f: matchall for f in State.fields})
    return State
    
    
class StateInfo (object):   # pylint: disable=R0903
    def __init__(self):
        self.untested_tnames = None
        self.islimit = None
        
    def __str__(self):
        return 'StateInfo: untested_t: #{}, islimit: {}'.format(
                len(self.untested_tnames, self.islimit))
                
        
class StatesInfo (dict):
    def __init__(self):
        dict.__init__(self)
        
    def istested(self, *args):
        if len(args) == 0:
            for s in self.values():
                if s.untested_tnames and s.islimit:
                    return False
            return True
        elif len(args) == 1:
            state = args[0]
            return state in self and not (self[state].untested_tnames and self[state].islimit)
        else:
            raise TypeError('istested expected at most 1 argument, got %s' % len(args))
            
    def reset(self, untested_tnames):
        for v in self.values():
            v.untested_tnames = untested_tnames[:]
            
    def get_unreached(self):
        return [s for s, si in self.items() if si.untested_tnames and si.islimit]
        
        
class Transition (object):
    def __init__(self, func=None, exprs=None, states=None):
        self.func = func
        self.exprs = {} if exprs is None else exprs
        self.states = {} if states is None else states
        
        
class Transitions (dict):
    def __init__(self):
        self.limits = []
        self.stateinfos = StatesInfo()
        self.path_to_untested = []
        
    def islimit(self, state):
        state = state.asdict()
        for limit in self.limits:
            try:
                if not eval(limit, state):
                    return False
            except Exception as e:  # pylint: disable=W0703
                log_error('error in limit:', e)
        return True
        
    def reset(self):
        self.stateinfos.reset(list(self.keys()))
        
    def add_stateinfo(self, state):
        try:
            return self.stateinfos[state]
        except KeyError:
            stateinfo = dict.setdefault(self.stateinfos, state, StateInfo())
            stateinfo.untested_tnames = list(self.keys())
            stateinfo.islimit = self.islimit(state)
            return stateinfo
            
    def update_transition(self, transition, state, target):
        transition.states[state] = target
        self.add_stateinfo(state)
        if target is not None and target not in transition.states:
            transition.states[target] = None
            self.add_stateinfo(target)
            
    @staticmethod
    def _path_to_list(path):
        result = []
        while path is not None:
            item, path = path
            result.append(item)
        return result
        
    def get_path_to_untested(self, state):
        paths = [[(state, None), None]]
        reachable = set()
        while paths:
            path = paths.pop(0)
            state = path[0][0]
            stateinfo = self.stateinfos[state]
            reachable.add(state)
            if stateinfo.islimit and stateinfo.untested_tnames:
                return self._path_to_list(path)
            for name, transition in self.items():
                target = transition.states[state]
                if not stateinfo.islimit or self.stateinfos[target].islimit:
                    if target not in reachable:
                        paths.append([(target, name), path])
        return None
        
    def get_random_transition(self, state):
        if not self.path_to_untested:
            stateinfo = self.stateinfos[state]
            while stateinfo.untested_tnames:
                name = stateinfo.untested_tnames.pop(random.randrange(len(stateinfo.untested_tnames)))
                states = self[name].states
                target = states.get(state, None)
                if target is None or self.stateinfos[target].islimit:
                    return name
            self.path_to_untested = self.get_path_to_untested(state)
            if self.path_to_untested is None:
                raise Quit('all reachable states tested')
            target, name = self.path_to_untested.pop()
            assert name is None
            assert target == state
        target, name = self.path_to_untested.pop()
        assert name is not None
        return name
        
        
class Result (OrderedDict):
    __slots__ = ()
    
    def __init__(self):
        OrderedDict.__init__(self)
        self._dict = dict(visited=0, states=0, transitions=0)
        
    errors = property(lambda self: sum(self.values()))
    
    def __str__(self):
        summary = '\n  visited: {0.visited}, states: {0.states}, transitions: {0.transitions}'.format(self)
        errors = self.errors
        if errors:
            summary += '\n  errors: {}'.format(errors)
            for k, v in self.items():
                summary += '\n    {} {}'.format(v, k)
        return summary
        
    def __getattr__(self, key):
        if key.startswith('_'):
            raise AttributeError()
        return self._dict[key]
        
    def __setattr__(self, key, value):
        if key.startswith('_'):
            OrderedDict.__setattr__(self, key, value)
        else:
            self._dict[key] = value
            
    def __iadd__(self, other):
        for k, v in other._dict.items():    # pylint: disable=W0212
            self._dict[k] += v
        for k, v in other.items():
            self[k] = self.get(k, 0) + v
        return self
        
        
class Quit (Exception): pass    # pylint: disable=C0321


class TestRunner (object):
    _instance = None
    lineno = None
    
    def __init__(self, testdata_dir, test, test_args):
        assert self._instance is None
        self.current_test = test
        self.write_mode = 'no'
        self.log_widgets = False
        for a in test_args:
            if a.startswith('write-n'):
                self.write_mode = 'no'
            elif a.startswith('write-y'):
                self.write_mode = 'yes'
            elif a.startswith('write-e'):
                self.write_mode = 'error'
            elif a == 'log-widgets':
                self.log_widgets = True
        self.current_testfile = os.path.join(testdata_dir, test)
        self.oneway_transitions = []
        self.transitions = Transitions()
        self.widgets = {}
        self.conditions = []
        self.known_widget_functions = []
        self.field_functions = []
        
    def run(self, main_window):
        self.main_window = main_window
        self.running = True
        settings.draw.speed = 120
        QTimer.singleShot(0, self.loop)
        
    def isvalid(self, state):
        state = state.asdict()
        for condition in self.conditions[:]:
            try:
                if not eval(condition, state):
                    return False
            except Exception as e:  # pylint: disable=W0703
                log_error('error in condition:', e)
                self.conditions.remove(condition)
        return True
        
    def read_test(self):
        def parse_fields(line):
            fields = OrderedDict()
            if line != 'Fields:\n':
                for f in sorted(utils.field_functions.keys()):
                    fields[f] = []
                return parse_conditions, line, [fields]
            return parse_field, None, [fields]
        def parse_field(line, fields):
            if not line.startswith('  '):
                return parse_conditions, line, [fields]
            field, value = line.split('=', 1)
            field = field.strip()
            if field not in list(utils.field_functions.keys()):
                log_error('unknown field:', field)
            else:
                value = eval(value, {})
                fields[field] = value
            return parse_field, None, [fields]
        def parse_conditions(line, fields):
            self.State = mkState(fields)
            self.fields = fields
            if line != 'Conditions:\n':
                return parse_limits, line, []
            return parse_condition, None, []
        def parse_condition(line):
            if not line.startswith('  '):
                return parse_limits, line, []
            self.conditions.append(line.strip())
            return parse_condition, None, []
        def parse_limits(line):
            if line != 'Limits:\n':
                return parse_initialstate, line, []
            return parse_limit, None, []
        def parse_limit(line):
            if not line.startswith('  '):
                return parse_initialstate, line, []
            self.transitions.limits.append(line.strip())
            return parse_limit, None, []
        def _parse_state_value(default, line, exprs=None):
            try:
                state = eval('dict({})'.format(line), {})
            except SyntaxError:
                logf_error('error parsing state:', '{!r}', line)
                state = None
            else:
                exprs = list((exprs or {}).keys())
                for field in list(state.keys()):
                    if field not in self.State.fields:
                        log_error('unselected field:', field)
                        del state[field]
                    elif field in exprs:
                        logf_error('state contains expression field', '{}: {}', field, line)
                state = default and default.replace(**state)
            return state
        def parse_initialstate(line):
            if not line.startswith('Initial-State:'):
                self.initial_state = None
                return parse_transition, line, []
            line = line.split(':', 1)[1].strip()
            self.initial_state = _parse_state_value(self.State.default, line)
            return parse_oneway, None, []
        def parse_oneway(line):
            if not line.startswith('One-Way:'):
                return parse_transition, line, []
            name = line.split(':', 1)[1].strip()
            transition = Transition()
            self.oneway_transitions.append((name, transition))
            return parse_state, None, [transition, True]
        def parse_transition(line):
            if not line.startswith('Transition:'):
                return parse_unknown, line, ['transition']
            name = line.split(':', 1)[1].strip()
            transition = Transition()
            if name in self.transitions:
                logf_error('duplicate transition:', '{!r}', name)
            else:
                self.transitions[name] = transition
            return parse_field_expr, None, [transition]
        def parse_field_expr(line, transition):
            if not line.startswith('  Expression:'):
                return parse_state, line, [transition, False]
            line = line.split(':', 1)[1].strip()
            try:
                field, expr = line.split('=', 1)
            except ValueError:
                logf_error('error parsing expression:', '{!r}', line)
            else:
                field = field.strip()
                expr = expr.strip()
                if field in transition.exprs:
                    logf_error('duplicate expression', 'for {}: {!r}', field, expr)
                elif field not in self.State.fields:
                    log_error('unknown expression field:', field)
                else:
                    transition.exprs[field] = expr
            return parse_field_expr, None, [transition]
        def _replace_target(exprs, state, target, field, value):
            if field in exprs:
                try:
                    value = eval(exprs[field], state.asdict())
                except Exception as e:  # pylint: disable=W0703
                    logf_error('error parsing expression', '{!r}: {}', exprs[field], e)
            target_value = getattr(target, field)
            if target_value is matchall:
                if value is matchall:
                    value = getattr(state, field)
                target = target.replace(**{field: value})
            else:
                if field in exprs:
                    if value == target_value:
                        log_error('target contains expression field:', field, 'value:', target_value)
                    else:
                        log_error('target contains expression field:', field, 'value:', target_value,
                                    'expected:', value)
            return target
        def _expand_state(fields, state, target, exprs, i=0):
            if state is None or target is None:
                return
            if i >= len(fields):
                yield state, target
                return
            field, values = fields[i]
            if getattr(state, field) is matchall:
                for value in values:
                    rstate = state.replace(**{field: value})
                    for estate, etarget in _expand_state(fields, rstate, target, exprs, i+1):
                        etarget = _replace_target(exprs, estate, etarget, field, value)
                        yield estate, etarget
            else:
                for estate, etarget in _expand_state(fields, state, target, exprs, i+1):
                    etarget = _replace_target(exprs, estate, etarget, field, matchall)
                    yield estate, etarget
        def parse_state(line, transition, oneway):
            if not line.startswith('  State:'):
                return parse_oneway if oneway else parse_transition, line, []
            line = line.split(':', 1)[1].strip()
            state = _parse_state_value(self.State.default, line)
            return parse_target, None, [transition, state, oneway]
        def parse_target(line, transition, state, oneway):
            if not line.startswith('        '):
                target = self.State.default
            else:
                target = _parse_state_value(self.State.default, line.strip(), transition.exprs)
                line = None
            for state, target in _expand_state(list(self.fields.items()), state, target, transition.exprs):
                if state in transition.states and transition.states[state] is not None:
                    log_error('duplicate state:', state.tostr())
                elif self.isvalid(state):
                    if not self.isvalid(target):
                        log_error('invalid target state:', target.tostr())
                        target = None
                    if oneway:
                        transition.states[state] = target
                    else:
                        self.transitions.update_transition(transition, state, target)
            return parse_state, line, [transition, oneway]
        def parse_unknown(line, src):
            if line == 'End.' or src == 'end':
                return parse_unknown, None, ['end']
            logf_error('error parsing', '{}: {!r}', src, line)
            return parse_transition, None, []
            
        def process_line(func, line, *args):
            while line is not None:
                func, line, args = func(line, *args)
            return func, args
            
        try:
            func = parse_fields
            args = []
            with open(self.current_testfile, 'rt', encoding='utf-8') as testfile:
                for logger.lineno, line in enumerate(testfile):
                    if line.lstrip().startswith('#'):
                        continue
                    func, args = process_line(func, line, *args)
        finally:
            process_line(func, 'End.', *args)
            logger.lineno = None
                    
    def _ignore_fields(self, states, igntype, exprs=None):
        result = {}
        for i, field in enumerate(self.State.fields):
            target_value = nomatch
            for state, target in states.items():
                if igntype == 'unchanged':
                    if state[i] != target[i]:
                        result = {}
                        break
                    state = state.replace(**{field: matchall})
                    target = target.replace(**{field: matchall})
                elif igntype == 'sametarget':
                    if target_value in {nomatch, target[i]}:
                        state = state.replace(**{field: matchall})
                        target_value = target[i]
                    else:
                        result = {}
                        break
                elif igntype == 'expressions':
                    if field in exprs:
                        target = target.replace(**{field: matchall})
                else:
                    assert False, igntype
                if state in result and result[state] != target:
                    result = {}
                    break
                result[state] = target
            else:
                states = result
                result = {}
        return states
        
    def write_test(self):
        with open(self.current_testfile, 'wt', encoding='utf-8') as testfile:
            testfile.write('Fields:\n')
            for field, values in self.fields.items():
                print(' ', field, '=', values, file=testfile)
            testfile.write('Conditions:\n')
            for condition in self.conditions:
                print(' ', condition, file=testfile)
            testfile.write('Limits:\n')
            for limit in self.transitions.limits:
                print(' ', limit, file=testfile)
            testfile.write('Initial-State: %s\n' % self.initial_state.tostr())
            for name, transition in self.oneway_transitions:
                testfile.write('One-Way: %s\n' % name)
                states = list(transition.states.items())
                #assert len(states) == 1
                for state, target in states:
                    testfile.write('  State: {}\n'.format(state.tostr()))
                    if state != target and target.tostr(state).strip():
                        testfile.write('         {}\n'.format(target.tostr(state)))
            for name, transition in sorted(self.transitions.items()):
                testfile.write('Transition: %s\n' % name)
                states = {s:t for s, t in transition.states.items() if self.transitions.stateinfos[s].islimit}
                states = self._ignore_fields(states, 'expressions', transition.exprs)
                states = self._ignore_fields(states, 'unchanged')
                states = self._ignore_fields(states, 'sametarget')
                for field, expr in sorted(transition.exprs.items()):
                    testfile.write('  Expression: {} = {}\n'.format(field, expr))
                for state, target in sorted(states.items()):
                    testfile.write('  State: {}\n'.format(state.tostr()))
                    if state != target and target.tostr(state).strip():
                        testfile.write('         {}\n'.format(target.tostr(state)))
                    
    def get_state(self):
        state = {field: func() for field, func in self.field_functions}
        return self.State(**state)
        
    def find_qobjects(self, root_widget, write_template=False):
        def log_obj(indent, msg, name, obj):
            if self.log_widgets:
                logf('{}{}: {} ({})', '  '*indent, msg, name, obj.__class__.__name__)
            
        objects = [(root_widget, 0)]
        while objects:
            obj, indent = objects.pop()
            name = obj.objectName()
            if name in self.widgets:
                log_obj(indent, 'kwnobj', name, obj)
            elif name and not name.startswith('qt_'):
                self.widgets[name] = obj
                if isinstance(obj, QAction):
                    for tname, transition in self.oneway_transitions:
                        if tname == name:
                            assert transition.func is None
                            transition.func = obj.trigger
                    if name in self.transitions:
                        transition = self.transitions[name]
                        assert transition.func is None
                        transition.func = obj.trigger
                    elif write_template:
                        self.transitions[name] = Transition(func=obj.trigger)
                log_obj(indent, 'object', name, obj)
            else:
                log_obj(indent, 'intobj', name, obj)
            for child in reversed(obj.children()):
                objects.append((child, indent+1))
                
    def update_field_functions(self):
        self.field_functions = []
        mk_func = lambda wname, func: lambda: func(self.widgets[wname])
        mk_default = lambda default: lambda: default
        for field, (wname, default, func) in utils.field_functions.items():
            if field in self.State.fields:
                if wname in self.widgets:
                    self.field_functions.append((field, mk_func(wname, func)))
                else:
                    self.field_functions.append((field, mk_default(default)))
        
    def update_transition_functions(self, write_template=False):
        def mk_transition(func, widgets, *args):
            args = tuple(widgets) + args
            return lambda: func(*args)
            
        for wnames, fname, func in utils.widget_functions:
            if fname in self.known_widget_functions:
                continue
            if not all((n in self.widgets) for n in wnames):
                continue
            self.known_widget_functions.append(fname)
            widgets = [self.widgets[n] for n in wnames]
            for tname, transition in self.oneway_transitions + list(self.transitions.items()):
                if ' ' in tname:
                    tfname, args = tname.split(' ', 1)
                    if tfname != fname:
                        continue
                    try:
                        args = eval(args, {'Qt': Qt})
                    except Exception as e:  # pylint: disable=W0703
                        logf_error('error parsing expression', '{!r}: {}', args, e)
                        args = None
                    else:
                        if type(args) is not tuple:
                            args = (args,)
                else:
                    if tname != fname:
                        continue
                    args = ()
                assert transition.func is None, (wnames, fname, transition.func)
                if args is None:
                    transition.func = lambda: None
                else:
                    transition.func = mk_transition(func, widgets, *args)
            if write_template:
                assert fname not in self.transitions
                self.transitions[fname] = Transition(func=mk_transition(func, widgets))
                
    def init_test(self):
        try:
            self.read_test()
        except IOError as e:
            log('Error reading test data file:', e)
        assert list(self.fields.keys()) == list(self.State.fields), (list(self.fields.keys()), self.State.fields)
        write_template = False
        if not self.transitions:
            log_error('empty test')
            write_template = True
            
        # introspect ui
        self.find_qobjects(self.main_window, write_template)
        self.update_field_functions()
        self.update_transition_functions(write_template)
        
        # initial state
        self.current_state = self.get_state()
        if self.initial_state is None:
            self.initial_state = self.current_state
            log_error('missing initial state:', self.initial_state.tostr())
        elif self.initial_state != self.current_state:
            logf_error('wrong initial state:', '\n     found: {}\n  expected: {}',
                        self.current_state.tostr(), self.initial_state.tostr())
            self.initial_state = self.current_state
            
        self.transitions.reset()
        if write_template:
            raise Quit('template created')
        
    def check_transition(self, name, transition, oneway=False):
        if oneway and len(transition.states) > 1:
            logf_error('ambiguous ow transition:', '\n  {}: {} -> ({})',
                            name, self.current_state.tostr(), len(transition.states))
            for state in list(transition.states.keys()):
                if state != self.current_state:
                    del transition.states[state]
        target = transition.states.get(self.current_state, None)
        current_state = self.get_state()
        if target != current_state:
            if target is None:
                logf_error('unknown transition:', '\n  {}: {} -> {}',
                            name, self.current_state.tostr(), current_state.tostr())
            else:
                logf_error('wrong target', 'for {}: {}\n     found: -> {}\n  expected: -> {}',
                            name, self.current_state.tostr(), current_state.tostr(), target.tostr())
            if oneway:
                transition.states[self.current_state] = current_state
            else:
                self.transitions.update_transition(transition, self.current_state, current_state)
        self.current_state = current_state
        
    def check_result(self):
        for name, transition in list(self.transitions.items()):
            if transition.func is None:
                log_error('unused transition:', name)
                del self.transitions[name]
        all_states = list(self.transitions.stateinfos.keys())
        all_transitions = [(name, state) for name, transition in self.transitions.items()
                                            for state in transition.states.keys()]
        logger.result.states = len(all_states)    # pylint: disable=W0201
        logger.result.transitions = len(all_transitions)  # pylint: disable=W0201
        unreached = self.transitions.stateinfos.get_unreached()
        for state in unreached:
            log_error('unreached', state.tostr())
        field_values = {}
        for name, transition in self.transitions.items():
            states = transition.states
            for state, target in list(states.items()):
                if state in unreached or not self.transitions.stateinfos[state].islimit or target is None:
                    del states[state]
                    continue
                for field, svalue, tvalue in zip(self.State.fields, state, target):
                    field_values.setdefault(field, set()).update([svalue, tvalue])
        for field, values in self.fields.items():
            if field in field_values:
                if set(values) != field_values[field]:
                    log_error('changed field values:', field)
                    values[:] = sorted(field_values[field])
            else:
                log_error('unused field', field)
                
    def step(self, name, transition):
        postfunc = transition.func()
        if postfunc == 'find_qobjects':
            self.find_qobjects(self.main_window)
            self.update_field_functions()
            self.update_transition_functions()
        logger.result.visited += 1    # pylint: disable=E1101
        yield
        while self.main_window.is_animating():
            yield
            
    def loop(self):
        for unused in self._iloop():
            if not self.main_window.isVisible():
                log_error('Unexpected end of test')
                self.running = False
                return
            QTest.qWait(10)
        # without this line sometimes there is a segfault in the next window
        self.main_window.deleteLater()
            
    def _iloop(self):
        self.current_state = name = target = None  # for except statement
        try:
            log('Initializing test:', self.current_test)
            self.init_test()
            yield
            log('Running test:', self.current_test)
            for name, transition in self.oneway_transitions:
                #TODO python3.3: yield from self.step(name, transition)
                for unused in self.step(name, transition):
                    yield
                self.check_transition(name, transition, True)
            if self.current_state not in self.transitions.stateinfos:
                log_error('unknown state:', self.current_state)
                stateinfo = self.transitions.add_stateinfo(self.current_state)
                if not stateinfo.islimit:
                    log_error('state not in limit:', self.current_state)
            while True:
                name = self.transitions.get_random_transition(self.current_state)
                transition = self.transitions[name]
                #TODO python3.3: yield from self.step(name, transition)
                for unused in self.step(name, transition):
                    yield
                self.check_transition(name, transition)
        except Quit as e:
            self.check_result()
            if self.write_mode == 'yes' or logger.result.errors and self.write_mode == 'error':
                self.write_test()
                log('test data file written')
            logf('End of test {}: {}', self.current_test, e)
        except Exception:
            self.check_result()
            logf('exception in {}: {} -> {}', name,
                    self.current_state and self.current_state.tostr(),
                    target and target.tostr())
            sys.excepthook(*sys.exc_info())
        finally:
            if self.running:
                self.running = False
                self.main_window.close()
        
    @classmethod
    def wrap(cls, testdata_dir, tests, test_args):
        logger.open('log-file' in test_args)
        result = Result()
        cnt_tests = 0
        for test in tests:
            if '=' in test:
                test, repeat = test.split('=', 1)
                repeat = int(repeat)
            else:
                repeat = 1
            cnt_tests += repeat
            for unused in range(repeat):
                logger.result = Result()
                instance = cls(testdata_dir, test, test_args)
                fileno, instance.settings_file = tempfile.mkstemp(suffix='-settings.conf', prefix='pybiktest-', text=True)
                os.close(fileno)
                yield instance
                if instance.running:
                    log_error('Unexpected end of testrunner')
                log('Result:', logger.result)
                log('')
                os.remove(instance.settings_file)
                del instance.settings_file
                result += logger.result
        if cnt_tests > 1:
            logf('Summary ({}): {}', cnt_tests, result)
        logger.close()