This file is indexed.

/usr/lib/python3/dist-packages/DistUpgrade/DistUpgradeController.py is in python3-distupgrade 1:0.220.10.

This file is owned by root:root, with mode 0o644.

The actual contents of the file can be viewed below.

   1
   2
   3
   4
   5
   6
   7
   8
   9
  10
  11
  12
  13
  14
  15
  16
  17
  18
  19
  20
  21
  22
  23
  24
  25
  26
  27
  28
  29
  30
  31
  32
  33
  34
  35
  36
  37
  38
  39
  40
  41
  42
  43
  44
  45
  46
  47
  48
  49
  50
  51
  52
  53
  54
  55
  56
  57
  58
  59
  60
  61
  62
  63
  64
  65
  66
  67
  68
  69
  70
  71
  72
  73
  74
  75
  76
  77
  78
  79
  80
  81
  82
  83
  84
  85
  86
  87
  88
  89
  90
  91
  92
  93
  94
  95
  96
  97
  98
  99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 138
 139
 140
 141
 142
 143
 144
 145
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
# DistUpgradeController.py 
#  
#  Copyright (c) 2004-2008 Canonical
#  
#  Author: Michael Vogt <michael.vogt@ubuntu.com>
# 
#  This program is free software; you can redistribute it and/or 
#  modify it under the terms of the GNU General Public License as 
#  published by the Free Software Foundation; either version 2 of the
#  License, or (at your option) any later version.
# 
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
# 
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
#  USA


from __future__ import absolute_import, print_function

import warnings
warnings.filterwarnings("ignore", "apt API not stable yet", FutureWarning)
import apt
import apt_pkg
import sys
import os
import subprocess
import locale
import logging
import shutil
import glob
import time
import copy
try:
    # >= 3.0
    from configparser import NoOptionError
    if sys.version >= '3.2':
        from configparser import ConfigParser as SafeConfigParser
    else:
        from configparser import SafeConfigParser
except ImportError:
    # < 3.0
    from ConfigParser import SafeConfigParser, NoOptionError
from .utils import (country_mirror,
                    url_downloadable,
                    check_and_fix_xbit,
                    get_arch,
                    iptables_active,
                    inside_chroot,
                    get_string_with_no_auth_from_source_entry,
                    is_child_of_process_name)
from string import Template
try:
    from urllib.parse import urlsplit
except ImportError:
    from urlparse import urlsplit

from .DistUpgradeView import STEP_PREPARE, STEP_MODIFY_SOURCES, STEP_FETCH, STEP_INSTALL, STEP_CLEANUP, STEP_REBOOT
from .DistUpgradeCache import MyCache
from .DistUpgradeConfigParser import DistUpgradeConfig
from .DistUpgradeQuirks import DistUpgradeQuirks
from .DistUpgradeAptCdrom import AptCdrom
from .DistUpgradeAufs import setupAufs, aufsOptionsAndEnvironmentSetup

# workaround broken relative import in python-apt (LP: #871007), we
# want the local version of distinfo.py from oneiric, but because of
# a bug in python-apt we will get the natty version that does not
# know about "Component.parent_component" leading to a crash
from . import distinfo
from . import sourceslist
sourceslist.DistInfo = distinfo.DistInfo

from .sourceslist import SourcesList, is_mirror
from .distro import get_distro, NoDistroTemplateException

from .DistUpgradeGettext import gettext as _
from .DistUpgradeGettext import ngettext
import gettext

from .DistUpgradeCache import (CacheExceptionDpkgInterrupted,
                               CacheExceptionLockingFailed,
                               NotEnoughFreeSpaceError)
from .DistUpgradeApport import run_apport

REBOOT_REQUIRED_FILE = "/var/run/reboot-required"


def component_ordering_key(a):
    """ key() function for sorted to ensure "correct" component ordering """
    ordering = ["main", "restricted", "universe", "multiverse"]
    try:
        return ordering.index(a)
    except ValueError:
        # ensure to sort behind the "official" components, order is not
        # really important for those
        return len(ordering)+1


class NoBackportsFoundException(Exception):
    pass


class DistUpgradeController(object):
    """ this is the controller that does most of the work """
    
    def __init__(self, distUpgradeView, options=None, datadir=None):
        # setup the paths
        localedir = "/usr/share/locale/"
        if datadir == None or datadir == '.':
            datadir = os.getcwd()
            localedir = os.path.join(datadir,"mo")
        self.datadir = datadir
        self.options = options

        # init gettext
        gettext.bindtextdomain("ubuntu-release-upgrader",localedir)
        gettext.textdomain("ubuntu-release-upgrader")

        # setup the view
        logging.debug("Using '%s' view" % distUpgradeView.__class__.__name__)
        self._view = distUpgradeView
        self._view.updateStatus(_("Reading cache"))
        self.cache = None

        if not self.options or self.options.withNetwork == None:
            self.useNetwork = True
        else:
            self.useNetwork = self.options.withNetwork
        if options:
            cdrompath = options.cdromPath
        else:
            cdrompath = None
        self.aptcdrom = AptCdrom(distUpgradeView, cdrompath)

        # the configuration
        self.config = DistUpgradeConfig(datadir)
        self.sources_backup_ext = "."+self.config.get("Files","BackupExt")

        # move some of the options stuff into the self.config, 
        # ConfigParser deals only with strings it seems *sigh*
        self.config.add_section("Options")
        self.config.set("Options","withNetwork", str(self.useNetwork))

        # aufs stuff
        aufsOptionsAndEnvironmentSetup(self.options, self.config)

        # some constants here
        self.fromDist = self.config.get("Sources","From")
        self.toDist = self.config.get("Sources","To")
        self.origin = self.config.get("Sources","ValidOrigin")
        self.arch = get_arch()

        # we run with --force-overwrite by default
        if "RELEASE_UPGRADE_NO_FORCE_OVERWRITE" not in os.environ:
            logging.debug("enable dpkg --force-overwrite")
            apt_pkg.config.set("DPkg::Options::","--force-overwrite")

        # we run in full upgrade mode by default
        self._partialUpgrade = False
        
        # install the quirks handler
        self.quirks = DistUpgradeQuirks(self, self.config)

        # setup env var 
        os.environ["RELEASE_UPGRADE_IN_PROGRESS"] = "1"
        os.environ["PYCENTRAL_FORCE_OVERWRITE"] = "1"
        os.environ["PATH"] = "%s:%s" % (os.getcwd()+"/imported",
                                        os.environ["PATH"])
        check_and_fix_xbit("./imported/invoke-rc.d")

        # set max retries
        maxRetries = self.config.getint("Network","MaxRetries")
        apt_pkg.config.set("Acquire::Retries", str(maxRetries))
        # max sizes for dpkgpm for large installs (see linux/limits.h and 
        #                                          linux/binfmts.h)
        apt_pkg.config.set("Dpkg::MaxArgs", str(64*1024))
        apt_pkg.config.set("Dpkg::MaxArgBytes", str(128*1024))

        # smaller to avoid hangs
        apt_pkg.config.set("Acquire::http::Timeout","20")
        apt_pkg.config.set("Acquire::ftp::Timeout","20")

        # no list cleanup here otherwise a "cancel" in the upgrade
        # will not restore the full state (lists will be missing)
        apt_pkg.config.set("Apt::Get::List-Cleanup", "false")

        # forced obsoletes
        self.forced_obsoletes = self.config.getlist("Distro","ForcedObsoletes")
        # list of valid mirrors that we can add
        self.valid_mirrors = self.config.getListFromFile("Sources","ValidMirrors")
        # third party mirrors
        self.valid_3p_mirrors = []
        if self.config.has_section('ThirdPartyMirrors'):
            self.valid_3p_mirrors = [pair[1] for pair in
                                     self.config.items('ThirdPartyMirrors')]
        # debugging
        #apt_pkg.config.set("DPkg::Options::","--debug=0077")

        # apt cron job
        self._aptCronJobPerms = 0o755

    def openCache(self, lock=True, restore_sources_list_on_fail=False):
        logging.debug("openCache()")
        if self.cache is None:
            self.quirks.run("PreCacheOpen")
        else:
            self.cache.release_lock()
            self.cache.unlock_lists_dir()
        # this loop will try getting the lock a couple of times
        MAX_LOCK_RETRIES = 20
        lock_retry = 0
        while True:
            try:
                # exit here once the cache is ready
                return self._openCache(lock)
            except CacheExceptionLockingFailed as e:
                # wait a bit
                lock_retry += 1
                self._view.processEvents()
                time.sleep(0.1)
                logging.debug(
                    "failed to lock the cache, retrying (%i)" % lock_retry)
                # and give up after some time
                if lock_retry > MAX_LOCK_RETRIES:
                    logging.error("Cache can not be locked (%s)" % e)
                    self._view.error(_("Unable to get exclusive lock"),
                                     _("This usually means that another "
                                       "package management application "
                                       "(like apt-get or aptitude) "
                                       "already running. Please close that "
                                       "application first."));
                    if restore_sources_list_on_fail:
                        self.abort()
                    else:
                        sys.exit(1)

    def _openCache(self, lock):
        try:
            self.cache = MyCache(self.config,
                                 self._view,
                                 self.quirks,
                                 self._view.getOpCacheProgress(),
                                 lock)
            # alias name for the plugin interface code
            self.apt_cache = self.cache
        # if we get a dpkg error that it was interrupted, just
        # run dpkg --configure -a
        except CacheExceptionDpkgInterrupted:
            logging.warning("dpkg interrupted, calling dpkg --configure -a")
            cmd = ["/usr/bin/dpkg","--configure","-a"]
            if os.environ.get("DEBIAN_FRONTEND") == "noninteractive":
                cmd.append("--force-confold")
            self._view.getTerminal().call(cmd)
            self.cache = MyCache(self.config,
                                 self._view,
                                 self.quirks,
                                 self._view.getOpCacheProgress())
        self.cache.partialUpgrade = self._partialUpgrade
        logging.debug("/openCache(), new cache size %i" % len(self.cache))

    def _viewSupportsSSH(self):
      """
      Returns True if this view support upgrades over ssh.
      In theory all views should support it, but for savety
      we do only allow text ssh upgrades (see LP: #322482)
      """
      supported = self.config.getlist("View","SupportSSH")
      if self._view.__class__.__name__ in supported:
          return True
      return False

    def _sshMagic(self):
        """ this will check for server mode and if we run over ssh.
            if this is the case, we will ask and spawn a additional
            daemon (to be sure we have a spare one around in case
            of trouble)
        """
        pidfile = os.path.join("/var/run/release-upgrader-sshd.pid")
        if (not os.path.exists(pidfile) and 
            os.path.isdir("/proc") and
            is_child_of_process_name("sshd")):
            # check if the frontend supports ssh upgrades (see lp: #322482)
            if not self._viewSupportsSSH():
                logging.error("upgrade over ssh not allowed")
                self._view.error(_("Upgrading over remote connection not supported"),
                                 _("You are running the upgrade over a "
                                   "remote ssh connection with a frontend "
                                   "that does "
                                   "not support this. Please try a text "
                                   "mode upgrade with 'do-release-upgrade'."
                                   "\n\n"
                                   "The upgrade will "
                                   "abort now. Please try without ssh.")
                                 )
                sys.exit(1)
                return False
            # ask for a spare one to start (and below 1024)
            port = 1022
            res = self._view.askYesNoQuestion(
                _("Continue running under SSH?"),
                _("This session appears to be running under ssh. "
                  "It is not recommended to perform a upgrade "
                  "over ssh currently because in case of failure "
                  "it is harder to recover.\n\n"
                  "If you continue, an additional ssh daemon will be "
                  "started at port '%s'.\n"
                  "Do you want to continue?") % port)
            # abort
            if res == False:
                sys.exit(1)
            res = subprocess.call(["/usr/sbin/sshd",
                                   "-o", "PidFile=%s" % pidfile,
                                   "-p",str(port)])
            if res == 0:
                summary = _("Starting additional sshd")
                descr =  _("To make recovery in case of failure easier, an "
                           "additional sshd will be started on port '%s'. "
                           "If anything goes wrong with the running ssh "
                           "you can still connect to the additional one.\n"
                           ) % port
                if iptables_active():
                    cmd = "iptables -I INPUT -p tcp --dport %s -j ACCEPT" % port
                    descr += _(
                        "If you run a firewall, you may need to "
                        "temporarily open this port. As this is "
                        "potentially dangerous it's not done automatically. "
                        "You can open the port with e.g.:\n'%s'") % cmd
                self._view.information(summary, descr)
        return True

    def _tryUpdateSelf(self):
        """ this is a helper that is run if we are started from a CD
            and we have network - we will then try to fetch a update
            of ourself
        """  
        from .MetaRelease import MetaReleaseCore
        from .DistUpgradeFetcherSelf import DistUpgradeFetcherSelf
        # check if we run from a LTS 
        forceLTS=False
        if (self.release == "dapper" or
            self.release == "hardy" or
            self.release == "lucid" or
            self.release == "precise"):
            forceLTS=True
        m = MetaReleaseCore(useDevelopmentRelease=False,
                            forceLTS=forceLTS)
        # this will timeout eventually
        self._view.processEvents()
        while not m.downloaded.wait(0.1):
            self._view.processEvents()
        if m.new_dist is None:
            logging.error("No new dist found")
            return False
        # we have a new dist
        progress = self._view.getAcquireProgress()
        fetcher = DistUpgradeFetcherSelf(new_dist=m.new_dist,
                                         progress=progress,
                                         options=self.options,
                                         view=self._view)
        fetcher.run()

    def _pythonSymlinkCheck(self):
        """ sanity check that /usr/bin/python points to the default
            python version. Users tend to modify this symlink, which
            breaks stuff in obscure ways (Ubuntu #75557).
        """
        logging.debug("_pythonSymlinkCheck run")
        if os.path.exists('/usr/share/python/debian_defaults'):
            config = SafeConfigParser()
            config.readfp(open('/usr/share/python/debian_defaults'))
            try:
                expected_default = config.get('DEFAULT', 'default-version')
            except NoOptionError:
                logging.debug("no default version for python found in '%s'" % config)
                return False
            try:
                fs_default_version = os.readlink('/usr/bin/python')
            except OSError as e:
                logging.error("os.readlink failed (%s)" % e)
                return False
            if not fs_default_version in (expected_default, os.path.join('/usr/bin', expected_default)):
                logging.debug("python symlink points to: '%s', but expected is '%s' or '%s'" % (fs_default_version, expected_default, os.path.join('/usr/bin', expected_default)))
                return False
        return True


    def prepare(self):
        """ initial cache opening, sanity checking, network checking """
        # first check if that is a good upgrade
        self.release = release = subprocess.Popen(["lsb_release","-c","-s"],
                                   stdout=subprocess.PIPE,
                                   universal_newlines=True).communicate()[0].strip()
        logging.debug("lsb-release: '%s'" % release)
        if not (release == self.fromDist or release == self.toDist):
            logging.error("Bad upgrade: '%s' != '%s' " % (release, self.fromDist))
            self._view.error(_("Can not upgrade"),
                             _("An upgrade from '%s' to '%s' is not "
                               "supported with this tool." % (release, self.toDist)))
            sys.exit(1)

        # setup aufs
        if self.config.getWithDefault("Aufs", "EnableFullOverlay", False):
            aufs_rw_dir = self.config.get("Aufs","RWDir")
            if not setupAufs(aufs_rw_dir):
                logging.error("aufs setup failed")
                self._view.error(_("Sandbox setup failed"),
                                 _("It was not possible to create the sandbox "
                                   "environment."))
                return False

            # all good, tell the user about the sandbox mode
            logging.info("running in aufs overlay mode")
            self._view.information(_("Sandbox mode"),
                                   _("This upgrade is running in sandbox "
                                     "(test) mode. All changes are written "
                                     "to '%s' and will be lost on the next "
                                     "reboot.\n\n"
                                     "*No* changes written to a system directory "
                                     "from now until the next reboot are "
                                     "permanent.") % aufs_rw_dir)

        # setup backports (if we have them)
        if self.options and self.options.havePrerequists:
            backportsdir = os.getcwd()+"/backports"
            logging.info("using backports in '%s' " % backportsdir)
            logging.debug("have: %s" % glob.glob(backportsdir+"/*.udeb"))
            if os.path.exists(backportsdir+"/usr/bin/dpkg"):
                apt_pkg.config.set("Dir::Bin::dpkg",backportsdir+"/usr/bin/dpkg");
            if os.path.exists(backportsdir+"/usr/lib/apt/methods"):
                apt_pkg.config.set("Dir::Bin::methods",backportsdir+"/usr/lib/apt/methods")
            conf = backportsdir+"/etc/apt/apt.conf.d/01ubuntu"
            if os.path.exists(conf):
                logging.debug("adding config '%s'" % conf)
                apt_pkg.read_config_file(apt_pkg.config, conf)

        # do the ssh check and warn if we run under ssh
        self._sshMagic()
        # check python version
        if not self._pythonSymlinkCheck():
            logging.error("pythonSymlinkCheck() failed, aborting")
            self._view.error(_("Can not upgrade"),
                             _("Your python install is corrupted. "
                               "Please fix the '/usr/bin/python' symlink."))
            sys.exit(1)
        # open cache
        try:
            self.openCache()
        except SystemError as e:
            logging.error("openCache() failed: '%s'" % e)
            return False
        if not self.cache.sanity_check(self._view):
            return False

        # now figure out if we need to go into desktop or 
        # server mode - we use a heuristic for this
        self.serverMode = self.cache.need_server_mode()
        if self.serverMode:
            os.environ["RELEASE_UPGRADE_MODE"] = "server"
        else:
            os.environ["RELEASE_UPGRADE_MODE"] = "desktop"

        if not self.checkViewDepends():
            logging.error("checkViewDepends() failed")
            return False

        if os.path.exists("/usr/bin/debsig-verify"):
            logging.error("debsig-verify is installed")
            self._view.error(_("Package 'debsig-verify' is installed"),
                             _("The upgrade can not continue with that "
                               "package installed.\n"
                               "Please remove it with synaptic "
                               "or 'apt-get remove debsig-verify' first "
                               "and run the upgrade again."))
            self.abort()

        from .DistUpgradeMain import SYSTEM_DIRS
        for systemdir in SYSTEM_DIRS:
            if os.path.exists(systemdir) and not os.access(systemdir, os.W_OK):
                logging.error("%s not writable" % systemdir)
                self._view.error(
                    _("Can not write to '%s'") % systemdir,
                    _("Its not possible to write to the system directory "
                      "'%s' on your system. The upgrade can not "
                      "continue.\n"
                      "Please make sure that the system directory is "
                      "writable.") % systemdir)
                self.abort()
            

        # FIXME: we may try to find out a bit more about the network
        # connection here and ask more  intelligent questions
        if self.aptcdrom and self.options and self.options.withNetwork == None:
            res = self._view.askYesNoQuestion(_("Include latest updates from the Internet?"),
                                              _("The upgrade system can use the internet to "
                                                "automatically download "
                                                "the latest updates and install them during the "
                                                "upgrade.  If you have a network connection this is "
                                                "highly recommended.\n\n"
                                                "The upgrade will take longer, but when "
                                                "it is complete, your system will be fully up to "
                                                "date.  You can choose not to do this, but you "
                                                "should install the latest updates soon after "
                                                "upgrading.\n"
                                                "If you answer 'no' here, the network is not "
                                                "used at all."),
                                              'Yes')
            self.useNetwork = res
            self.config.set("Options","withNetwork", str(self.useNetwork))
            logging.debug("useNetwork: '%s' (selected by user)" % res)
            if res:
                self._tryUpdateSelf()
        return True

    def _sourcesListEntryDownloadable(self, entry):
        """
        helper that checks if a sources.list entry points to 
        something downloadable
        """
        logging.debug("verifySourcesListEntry: %s" % entry)
        # no way to verify without network
        if not self.useNetwork:
            logging.debug("skiping downloadable check (no network)")
            return True
        # check if the entry points to something we can download
        uri = "%s/dists/%s/Release" % (entry.uri, entry.dist)
        return url_downloadable(uri, logging.debug)

    def rewriteSourcesList(self, mirror_check=True):
        logging.debug("rewriteSourcesList()")

        sync_components = self.config.getlist("Sources","Components")

        # skip mirror check if special environment is set
        # (useful for server admins with internal repos)
        if (self.config.getWithDefault("Sources","AllowThirdParty",False) or
            "RELEASE_UPRADER_ALLOW_THIRD_PARTY" in os.environ):
            logging.warning("mirror check skipped, *overriden* via config")
            mirror_check=False

        # check if we need to enable main
        if mirror_check == True and self.useNetwork:
            # now check if the base-meta pkgs are available in
            # the archive or only available as "now"
            # -> if not that means that "main" is missing and we
            #    need to  enable it
            for pkgname in self.config.getlist("Distro","BaseMetaPkgs"):
                if ((not pkgname in self.cache or
                     not self.cache[pkgname].candidate or
                     len(self.cache[pkgname].candidate.origins) == 0)
                    or
                    (self.cache[pkgname].candidate and
                     len(self.cache[pkgname].candidate.origins) == 1 and
                     self.cache[pkgname].candidate.origins[0].archive == "now")
                   ):
                    logging.debug("BaseMetaPkg '%s' has no candidate.origins" % pkgname)
                    try:
                        distro = get_distro()
                        distro.get_sources(self.sources)
                        distro.enable_component("main")
                    except NoDistroTemplateException:
                        # fallback if everything else does not work,
                        # we replace the sources.list with a single
                        # line to ubuntu-main
                        logging.warning('get_distro().enable_component("main") failed, overwriting sources.list instead as last resort')
                        s =  "# auto generated by ubuntu-release-upgrader"
                        s += "deb http://archive.ubuntu.com/ubuntu %s main restricted" % self.toDist
                        s += "deb http://archive.ubuntu.com/ubuntu %s-updates main restricted" % self.toDist
                        s += "deb http://security.ubuntu.com/ubuntu %s-security main restricted" % self.toDist
                        open("/etc/apt/sources.list","w").write(s)
                    break

        # this must map, i.e. second in "from" must be the second in "to"
        # (but they can be different, so in theory we could exchange
        #  component names here)
        pockets = self.config.getlist("Sources","Pockets")
        fromDists = [self.fromDist] + ["%s-%s" % (self.fromDist, x)
                                       for x in pockets]
        toDists = [self.toDist] + ["%s-%s" % (self.toDist,x)
                                   for x in pockets]
        self.sources_disabled = False

        # look over the stuff we have
        foundToDist = False
        # collect information on what components (main,universe) are enabled for what distro (sub)version
        # e.g. found_components = { 'hardy':set("main","restricted"), 'hardy-updates':set("main") }
        self.found_components = {}
        for entry in self.sources.list[:]:

            # ignore invalid records or disabled ones
            if entry.invalid or entry.disabled:
                continue

            # we disable breezy cdrom sources to make sure that demoted
            # packages are removed
            if entry.uri.startswith("cdrom:") and entry.dist == self.fromDist:
                logging.debug("disabled '%s' cdrom entry (dist == fromDist)" % entry)
                entry.disabled = True
                continue
            # check if there is actually a lists file for them available
            # and disable them if not
            elif entry.uri.startswith("cdrom:"):
                # 
                listdir = apt_pkg.config.find_dir("Dir::State::lists")
                if not os.path.exists("%s/%s%s_%s_%s" % 
                                      (listdir,
                                       apt_pkg.uri_to_filename(entry.uri),
                                       "dists",
                                       entry.dist,
                                       "Release")):
                    logging.warning("disabling cdrom source '%s' because it has no  Release file" % entry)
                    entry.disabled = True
                continue

            # special case for archive.canonical.com that needs to
            # be rewritten (for pre-gutsy upgrades)
            cdist = "%s-commercial" % self.fromDist
            if (not entry.disabled and
                entry.uri.startswith("http://archive.canonical.com") and
                entry.dist == cdist):
                entry.dist = self.toDist
                entry.comps = ["partner"]
                logging.debug("transitioned commercial to '%s' " % entry)
                continue

            # special case for landscape.canonical.com because they
            # don't use a standard archive layout (gutsy->hardy)
            if (not entry.disabled and
                entry.uri.startswith("http://landscape.canonical.com/packages/%s" % self.fromDist)):
                logging.debug("commenting landscape.canonical.com out")
                entry.disabled = True
                continue

            # Disable proposed on upgrade to a development release.
            if (not entry.disabled and self.options
                and self.options.devel_release == True and
                "%s-proposed" % self.fromDist in entry.dist):
                logging.debug("upgrade to development release, disabling proposed")
                entry.dist = "%s-proposed" % self.toDist
                entry.comment += _("Not for humans during development stage of release %s") % self.toDist
                entry.disabled = True
                continue

            # handle upgrades from a EOL release and check if there
            # is a supported release available
            if (not entry.disabled and
                "old-releases.ubuntu.com/" in entry.uri):
                logging.debug("upgrade from old-releases.ubuntu.com detected")
                # test country mirror first, then archive.u.c
                for uri in ["http://%sarchive.ubuntu.com/ubuntu" % country_mirror(),
                            "http://archive.ubuntu.com/ubuntu"]:
                    test_entry = copy.copy(entry)
                    test_entry.uri = uri
                    test_entry.dist = self.toDist
                    if self._sourcesListEntryDownloadable(test_entry):
                        logging.info("transition from old-release.u.c to %s" % uri)
                        entry.uri = uri
                        break

            logging.debug("examining: '%s'" %  get_string_with_no_auth_from_source_entry(entry))
            # check if it's a mirror (or official site)
            validMirror = self.isMirror(entry.uri)
            thirdPartyMirror = not mirror_check or self.isThirdPartyMirror(entry.uri)
            if validMirror or thirdPartyMirror:
                # disabled/security/commercial/extras are special cases
                # we use validTo/foundToDist to figure out if we have a 
                # main archive mirror in the sources.list or if we 
                # need to add one
                validTo = True
                if (entry.disabled or
                    entry.type == "deb-src" or
                    "/security.ubuntu.com" in entry.uri or
                    "%s-security" % self.fromDist in entry.dist or
                    "%s-backports" % self.fromDist in entry.dist or
                    "/archive.canonical.com" in entry.uri or
                    "/extras.ubuntu.com" in entry.uri):
                    validTo = False
                if entry.dist in toDists:
                    # so the self.sources.list is already set to the new
                    # distro
                    logging.debug("entry '%s' is already set to new dist" % get_string_with_no_auth_from_source_entry(entry))
                    foundToDist |= validTo
                elif entry.dist in fromDists:
                    foundToDist |= validTo
                    entry.dist = toDists[fromDists.index(entry.dist)]
                    logging.debug("entry '%s' updated to new dist" % get_string_with_no_auth_from_source_entry(entry))
                elif entry.type == 'deb-src':
                    continue
                elif validMirror:
                    # disable all entries that are official but don't
                    # point to either "to" or "from" dist
                    entry.disabled = True
                    self.sources_disabled = True
                    logging.debug("entry '%s' was disabled (unknown dist)" % get_string_with_no_auth_from_source_entry(entry))

                # if we make it to this point, we have a official or third-party mirror

                # check if the arch is powerpc or sparc and if so, transition
                # to ports.ubuntu.com (powerpc got demoted in gutsy, sparc
                # in hardy)
                if (entry.type == "deb" and
                    not "ports.ubuntu.com" in entry.uri and
                    (self.arch == "powerpc" or self.arch == "sparc")):
                    logging.debug("moving %s source entry to 'ports.ubuntu.com' " % self.arch)
                    entry.uri = "http://ports.ubuntu.com/ubuntu-ports/"

                # gather what components are enabled and are inconsistent
                for d in ["%s" % self.toDist,
                          "%s-updates" % self.toDist,
                          "%s-security" % self.toDist]:
                    # create entry if needed, ignore disabled
                    # entries and deb-src
                    self.found_components.setdefault(d,set())
                    if (not entry.disabled and entry.dist == d and
                        entry.type == "deb"):
                        for comp in entry.comps:
                            # only sync components we know about
                            if not comp in sync_components:
                                continue
                            self.found_components[d].add(comp)

            else:
                # disable anything that is not from a official mirror or a whitelisted third party
                if entry.dist == self.fromDist:
                    entry.dist = self.toDist
                disable_comment = " " + _("disabled on upgrade to %s") % self.toDist
                if isinstance(entry.comment, bytes):
                    entry.comment += disable_comment.encode('UTF-8')
                else:
                    entry.comment += disable_comment
                entry.disabled = True
                self.sources_disabled = True
                logging.debug("entry '%s' was disabled (unknown mirror)" % get_string_with_no_auth_from_source_entry(entry))

        # now go over the list again and check for missing components
        # in $dist-updates and $dist-security and add them
        for entry in self.sources.list[:]:
            # skip all comps that are not relevant (including e.g. "hardy")
            if (entry.invalid or entry.disabled or entry.type == "deb-src" or 
                entry.uri.startswith("cdrom:") or entry.dist == self.toDist):
                continue
            # now check for "$dist-updates" and "$dist-security" and add any inconsistencies
            if entry.dist in self.found_components:
                component_diff = self.found_components[self.toDist]-self.found_components[entry.dist]
                if component_diff:
                    logging.info("fixing components inconsistency from '%s'" % get_string_with_no_auth_from_source_entry(entry))
                    # extend and make sure to keep order
                    entry.comps.extend(
                        sorted(component_diff, key=component_ordering_key))
                    logging.info("to new entry '%s'" % get_string_with_no_auth_from_source_entry(entry))
                    del self.found_components[entry.dist]
        return foundToDist

    def updateSourcesList(self):
        logging.debug("updateSourcesList()")
        self.sources = SourcesList(matcherPath=".")
        # backup first!
        self.sources.backup(self.sources_backup_ext)
        if not self.rewriteSourcesList(mirror_check=True):
            logging.error("No valid mirror found")
            res = self._view.askYesNoQuestion(_("No valid mirror found"),
                             _("While scanning your repository "
                               "information no mirror entry for "
                               "the upgrade was found. "
                               "This can happen if you run an internal "
                               "mirror or if the mirror information is "
                               "out of date.\n\n"
                               "Do you want to rewrite your "
                               "'sources.list' file anyway? If you choose "
                               "'Yes' here it will update all '%s' to '%s' "
                               "entries.\n"
                               "If you select 'No' the upgrade will cancel."
                               ) % (self.fromDist, self.toDist))
            if res:
                # re-init the sources and try again
                self.sources = SourcesList(matcherPath=".")
                # its ok if rewriteSourcesList fails here if
                # we do not use a network, the sources.list may be empty
                if (not self.rewriteSourcesList(mirror_check=False)
                    and self.useNetwork):
                    #hm, still nothing useful ...
                    prim = _("Generate default sources?")
                    secon = _("After scanning your 'sources.list' no "
                              "valid entry for '%s' was found.\n\n"
                              "Should default entries for '%s' be "
                              "added? If you select 'No', the upgrade "
                              "will cancel.") % (self.fromDist, self.toDist)
                    if not self._view.askYesNoQuestion(prim, secon):
                        self.abort()

                    # add some defaults here
                    # FIXME: find mirror here
                    logging.info("generate new default sources.list")
                    uri = "http://archive.ubuntu.com/ubuntu"
                    comps = ["main","restricted"]
                    self.sources.add("deb", uri, self.toDist, comps)
                    self.sources.add("deb", uri, self.toDist+"-updates", comps)
                    self.sources.add("deb",
                                     "http://security.ubuntu.com/ubuntu/",
                                     self.toDist+"-security", comps)
            else:
                self.abort()

        # now write
        self.sources.save()

        # re-check if the written self.sources are valid, if not revert and
        # bail out
        # TODO: check if some main packages are still available or if we
        #       accidentally shot them, if not, maybe offer to write a standard
        #       sources.list?
        try:
            sourceslist = apt_pkg.SourceList()
            sourceslist.read_main_list()
        except SystemError:
            logging.error("Repository information invalid after updating (we broke it!)")
            if os.path.exists("/usr/bin/apport-bug"):
                self._view.error(_("Repository information invalid"),
                                 _("Upgrading the repository information "
                                   "resulted in a invalid file so a bug "
                                   "reporting process is being started."))
                subprocess.Popen(["apport-bug", "ubuntu-release-upgrader-core"])
            else:
                self._view.error(_("Repository information invalid"),
                                 _("Upgrading the repository information "
                                   "resulted in a invalid file. To report "
                                   "a bug install apport and then execute "
                                   "'apport-bug ubuntu-release-upgrader'."))
                logging.error("Missing apport-bug, bug report not "
                              "autocreated")
            return False

        if self.sources_disabled:
            self._view.information(_("Third party sources disabled"),
                             _("Some third party entries in your sources.list "
                               "were disabled. You can re-enable them "
                               "after the upgrade with the "
                               "'software-properties' tool or "
                               "your package manager."
                               ))
        return True

    def _logChanges(self):
        # debugging output
        logging.debug("About to apply the following changes")
        inst = []
        up = []
        rm = []
        held = []
        keep = []
        for pkg in self.cache:
            if pkg.marked_install: inst.append(pkg.name)
            elif pkg.marked_upgrade: up.append(pkg.name)
            elif pkg.marked_delete: rm.append(pkg.name)
            elif (pkg.is_installed and pkg.is_upgradable): held.append(pkg.name)
            elif pkg.is_installed and pkg.marked_keep: keep.append(pkg.name)
        logging.debug("Keep at same version: %s" % " ".join(keep))
        logging.debug("Upgradable, but held- back: %s" % " ".join(held))
        logging.debug("Remove: %s" % " ".join(rm))
        logging.debug("Install: %s" % " ".join(inst))
        logging.debug("Upgrade: %s" % " ".join(up))
        

    def doPostInitialUpdate(self):
        # check if we have packages in ReqReinst state that are not
        # downloadable
        logging.debug("doPostInitialUpdate")
        self.quirks.run("PostInitialUpdate")
        if not self.cache:
            return False
        if len(self.cache.req_reinstall_pkgs) > 0:
            logging.warning("packages in reqReinstall state, trying to fix")
            self.cache.fix_req_reinst(self._view)
            self.openCache()
        if len(self.cache.req_reinstall_pkgs) > 0:
            reqreinst = self.cache.req_reinstall_pkgs
            header = ngettext("Package in inconsistent state",
                              "Packages in inconsistent state",
                              len(reqreinst))
            summary = ngettext("The package '%s' is in an inconsistent "
                               "state and needs to be reinstalled, but "
                               "no archive can be found for it. "
                               "Please reinstall the package manually "
                               "or remove it from the system.",
                               "The packages '%s' are in an inconsistent "
                               "state and need to be reinstalled, but "
                               "no archive can be found for them. "
                               "Please reinstall the packages manually "
                               "or remove them from the system.",
                               len(reqreinst)) % ", ".join(reqreinst)
            self._view.error(header, summary)
            return False
        # FIXME: check out what packages are downloadable etc to
        # compare the list after the update again
        self.obsolete_pkgs = self.cache._getObsoletesPkgs()
        self.foreign_pkgs = self.cache._getForeignPkgs(self.origin, self.fromDist, self.toDist)
        if self.serverMode:
            self.tasks = self.cache.installedTasks
        logging.debug("Foreign: %s" % " ".join(self.foreign_pkgs))
        logging.debug("Obsolete: %s" % " ".join(self.obsolete_pkgs))
        return True

    def doUpdate(self, showErrors=True, forceRetries=None):
        logging.debug("running doUpdate() (showErrors=%s)" % showErrors)
        if not self.useNetwork:
            logging.debug("doUpdate() will not use the network because self.useNetwork==false")
            return True
        self.cache._list.read_main_list()
        progress = self._view.getAcquireProgress()
        # FIXME: also remove all files from the lists partial dir!
        currentRetry = 0
        if forceRetries is not None:
            maxRetries=forceRetries
        else:
            maxRetries = self.config.getint("Network","MaxRetries")
        while currentRetry < maxRetries:
            try:
                self.cache.update(progress)
            except (SystemError, IOError) as e:
                logging.error("IOError/SystemError in cache.update(): '%s'. Retrying (currentRetry: %s)" % (e,currentRetry))
                currentRetry += 1
                continue
            # no exception, so all was fine, we are done
            return True

        logging.error("doUpdate() failed completely")
        if showErrors:
            self._view.error(_("Error during update"),
                             _("A problem occurred during the update. "
                               "This is usually some sort of network "
                               "problem, please check your network "
                               "connection and retry."), "%s" % e)
        return False


    def _checkFreeSpace(self):
        " this checks if we have enough free space on /var and /usr"
        err_sum = _("Not enough free disk space")
        err_long= _("The upgrade has aborted. "
                    "The upgrade needs a total of %s free space on disk '%s'. "
                    "Please free at least an additional %s of disk "
                    "space on '%s'. "
                    "Empty your trash and remove temporary "
                    "packages of former installations using "
                    "'sudo apt-get clean'.")
        # allow override
        if self.config.getWithDefault("FreeSpace","SkipCheck",False):
            logging.warning("free space check skipped via config override")
            return True
        # do the check
        with_snapshots = self._is_apt_btrfs_snapshot_supported()
        try:
            self.cache.checkFreeSpace(with_snapshots)
        except NotEnoughFreeSpaceError as e:
            # ok, showing multiple error dialog sucks from the UI
            # perspective, but it means we do not need to break the
            # string freeze
            for required in e.free_space_required_list:
                self._view.error(err_sum, err_long % (required.size_total,
                                                      required.dir,
                                                      required.size_needed,
                                                      required.dir))
            return False
        return True


    def askDistUpgrade(self):
        self._view.updateStatus(_("Calculating the changes"))
        if not self.cache.distUpgrade(self._view, self.serverMode, self._partialUpgrade):
            return False

        if self.serverMode:
            if not self.cache.installTasks(self.tasks):
                return False

        # show changes and confirm
        changes = self.cache.get_changes()
        self._view.processEvents()

        # log the changes for debugging
        self._logChanges()
        self._view.processEvents()

        # check if we have enough free space 
        if not self._checkFreeSpace():
            return False
        self._view.processEvents()

        # get the demotions
        self.installed_demotions = self.cache.get_installed_demoted_packages()
        if len(self.installed_demotions) > 0:
            self.installed_demotions.sort()
            logging.debug("demoted: '%s'" % " ".join([x.name for x in self.installed_demotions]))
            logging.debug("found components: %s" % self.found_components)

        # flush UI
        self._view.processEvents()

        # ask the user
        res = self._view.confirmChanges(_("Do you want to start the upgrade?"),
                                        changes,
                                        self.installed_demotions,
                                        self.cache.required_download)
        return res

    def _disableAptCronJob(self):
        if os.path.exists("/etc/cron.daily/apt"):
            #self._aptCronJobPerms = os.stat("/etc/cron.daily/apt")[ST_MODE]
            logging.debug("disabling apt cron job (%s)" % oct(self._aptCronJobPerms))
            os.chmod("/etc/cron.daily/apt",0o644)
    def _enableAptCronJob(self):
        if os.path.exists("/etc/cron.daily/apt"):
            logging.debug("enabling apt cron job")
            os.chmod("/etc/cron.daily/apt", self._aptCronJobPerms)

    def doDistUpgradeFetching(self):
        # ensure that no apt cleanup is run during the download/install
        self._disableAptCronJob()
        # get the upgrade
        currentRetry = 0
        fprogress = self._view.getAcquireProgress()
        #iprogress = self._view.getInstallProgress(self.cache)
        # start slideshow
        url = self.config.getWithDefault("Distro","SlideshowUrl",None)
        if url:
            try:
                lang = locale.getdefaultlocale()[0].split('_')[0]
            except:
                logging.exception("getdefaultlocale")
                lang = "en"
            self._view.getHtmlView().open("%s#locale=%s" % (url, lang))
        # retry the fetching in case of errors
        maxRetries = self.config.getint("Network","MaxRetries")
        # FIXME: we get errors like 
        #   "I wasn't able to locate file for the %s package" 
        #  here sometimes. its unclear why and not reproducible, the 
        #  current theory is that for some reason the file is not
        #  considered trusted at the moment 
        #  pkgAcquireArchive::QueueNext() runs debReleaseIndex::IsTrused()
        #  (the later just checks for the existence of the .gpg file)
        #  OR 
        #  the fact that we get a pm and fetcher here confuses something
        #  in libapt?
        # POSSIBLE workaround: keep the list-dir locked so that 
        #          no apt-get update can run outside from the release
        #          upgrader 
        user_canceled = False
        # LP: #1102593 - In Python 3, the targets of except clauses get `del`d
        # from the current namespace after the exception is handled, so we
        # must assign it to a different variable in order to use it after
        # the while loop.
        exception = None
        while currentRetry < maxRetries:
            try:
                pm = apt_pkg.PackageManager(self.cache._depcache)
                fetcher = apt_pkg.Acquire(fprogress)
                self.cache._fetch_archives(fetcher, pm)
            except apt.cache.FetchCancelledException as e:
                logging.info("user canceled")
                user_canceled = True
                exception = e
                break
            except IOError as e:
                # fetch failed, will be retried
                logging.error("IOError in cache.commit(): '%s'. Retrying (currentTry: %s)" % (e,currentRetry))
                currentRetry += 1
                exception = e
                continue
            return True

        # maximum fetch-retries reached without a successful commit
        if user_canceled:
            self._view.information(_("Upgrade canceled"),
                                   _("The upgrade will cancel now and the "
                                     "original system state will be restored. "
                                     "You can resume the upgrade at a later "
                                     "time."))
        else:
            logging.error("giving up on fetching after maximum retries")
            self._view.error(_("Could not download the upgrades"),
                             _("The upgrade has aborted. Please check your "
                               "Internet connection or "
                               "installation media and try again. All files "
                               "downloaded so far have been kept."),
                             "%s" % exception)
        # abort here because we want our sources.list back
        self._enableAptCronJob()
        self.abort()

    def _is_apt_btrfs_snapshot_supported(self):
        """ check if apt-btrfs-snapshot is usable """
        try:
            import apt_btrfs_snapshot
        except ImportError:
            return
        try:
            apt_btrfs = apt_btrfs_snapshot.AptBtrfsSnapshot()
            res = apt_btrfs.snapshots_supported()
        except:
            logging.exception("failed to check btrfs support")
            return False
        logging.debug("apt btrfs snapshots supported: %s" % res)
        return res

    def _maybe_create_apt_btrfs_snapshot(self):
        """ create btrfs snapshot (if btrfs layout is there) """
        if not self._is_apt_btrfs_snapshot_supported():
            return
        import apt_btrfs_snapshot
        apt_btrfs = apt_btrfs_snapshot.AptBtrfsSnapshot()
        prefix = "release-upgrade-%s-" % self.toDist
        res = apt_btrfs.create_btrfs_root_snapshot(prefix)
        logging.info("creating snapshot '%s' (success=%s)" % (prefix, res))

    def doDistUpgrade(self):
        # add debug code only here
        #apt_pkg.config.set("Debug::pkgDpkgPM", "1")
        #apt_pkg.config.set("Debug::pkgOrderList", "1")
        #apt_pkg.config.set("Debug::pkgPackageManager", "1")

        # get the upgrade
        currentRetry = 0
        fprogress = self._view.getAcquireProgress()
        iprogress = self._view.getInstallProgress(self.cache)
        # retry the fetching in case of errors
        maxRetries = self.config.getint("Network","MaxRetries")
        if not self._partialUpgrade:
            self.quirks.run("StartUpgrade")
            # FIXME: take this into account for diskspace calculation
            self._maybe_create_apt_btrfs_snapshot()
        res = False
        while currentRetry < maxRetries:
            try:
                res = self.cache.commit(fprogress,iprogress)
                logging.debug("cache.commit() returned %s" % res)
            except SystemError as e:
                logging.error("SystemError from cache.commit(): %s" % e)
                # if its a ordering bug we can cleanly revert to
                # the previous release, no packages have been installed
                # yet (LP: #328655, #356781)
                if os.path.exists("/var/run/ubuntu-release-upgrader-apt-exception"):
                    e = open("/var/run/ubuntu-release-upgrader-apt-exception").read()
                    logging.error("found exception: '%s'" % e)
                    # if its a ordering bug we can cleanly revert but we need to write
                    # a marker for the parent process to know its this kind of error
                    pre_configure_errors = [
                        "E:Internal Error, Could not perform immediate configuration",
                        "E:Couldn't configure pre-depend "]
                    for preconf_error in pre_configure_errors:
                        if str(e).startswith(preconf_error):
                            logging.debug("detected preconfigure error, restorting state")
                            self._enableAptCronJob()
                            # FIXME: strings are not good, but we are in string freeze
                            # currently
                            msg = _("Error during commit")
                            msg += "\n'%s'\n" % str(e)
                            msg += _("Restoring original system state")
                            self._view.error(_("Could not install the upgrades"), msg)
                            # abort() exits cleanly
                            self.abort()
                
                # invoke the frontend now and show a error message
                msg = _("The upgrade has aborted. Your system "
                        "could be in an unusable state. A recovery "
                        "will run now (dpkg --configure -a).")
                if not self._partialUpgrade:
                    if not run_apport():
                        msg += _("\n\nPlease report this bug in a browser at "
                                 "http://bugs.launchpad.net/ubuntu/+source/ubuntu-release-upgrader/+filebug "
                                 "and attach the files in /var/log/dist-upgrade/ "
                                 "to the bug report.\n"
                                 "%s" % e)
                self._view.error(_("Could not install the upgrades"), msg)
                # installing the packages failed, can't be retried
                cmd = ["/usr/bin/dpkg","--configure","-a"]
                if os.environ.get("DEBIAN_FRONTEND") == "noninteractive":
                    cmd.append("--force-confold")
                self._view.getTerminal().call(cmd)
                self._enableAptCronJob()
                return False
            except IOError as e:
                # fetch failed, will be retried
                logging.error("IOError in cache.commit(): '%s'. Retrying (currentTry: %s)" % (e,currentRetry))
                currentRetry += 1
                continue
            except OSError as e:
                logging.exception("cache.commit()")
                # deal gracefully with:
                #  OSError: [Errno 12] Cannot allocate memory
                if e.errno == 12:
                    self._enableAptCronJob()
                    msg = _("Error during commit")
                    msg += "\n'%s'\n" % str(e)
                    msg += _("Restoring original system state")
                    self._view.error(_("Could not install the upgrades"), msg)
                    # abort() exits cleanly
                    self.abort()
            # no exception, so all was fine, we are done
            self._enableAptCronJob()
            return True
        
        # maximum fetch-retries reached without a successful commit
        logging.error("giving up on fetching after maximum retries")
        self._view.error(_("Could not download the upgrades"),
                         _("The upgrade has aborted. Please check your "\
                           "Internet connection or "\
                           "installation media and try again. "),
                           "%s" % e)
        # abort here because we want our sources.list back
        self.abort()

    def doPostUpgrade(self):
        # reopen cache
        self.openCache()
        # run the quirks handler that does does like things adding
        # missing groups or similar work arounds, only do it on real
        # upgrades
        self.quirks.run("PostUpgrade")
        # check out what packages are cruft now
        # use self.{foreign,obsolete}_pkgs here and see what changed
        now_obsolete = self.cache._getObsoletesPkgs()
        now_foreign = self.cache._getForeignPkgs(self.origin, self.fromDist, self.toDist)
        logging.debug("Obsolete: %s" % " ".join(now_obsolete))
        logging.debug("Foreign: %s" % " ".join(now_foreign))
        # now sanity check - if a base meta package is in the obsolete list now, that means
        # that something went wrong (see #335154) badly with the network. this should never happen, but it did happen
        # at least once so we add extra paranoia here
        for pkg in self.config.getlist("Distro","BaseMetaPkgs"):
            if pkg in now_obsolete:
                logging.error("the BaseMetaPkg '%s' is in the obsolete list, something is wrong, ignoring the obsoletes" % pkg)
                now_obsolete = set()
                break
        # check if we actually want obsolete removal
        if not self.config.getWithDefault("Distro","RemoveObsoletes", True):
            logging.debug("Skipping obsolete Removal")
            return True

        # now get the meta-pkg specific obsoletes and purges
        for pkg in self.config.getlist("Distro","MetaPkgs"):
            if pkg in self.cache and self.cache[pkg].is_installed:
                self.forced_obsoletes.extend(self.config.getlist(pkg,"ForcedObsoletes"))
        # now add the obsolete kernels to the forced obsoletes
        self.forced_obsoletes.extend(self.cache.identifyObsoleteKernels())
        logging.debug("forced_obsoletes: %s", self.forced_obsoletes)

        # mark packages that are now obsolete (and where not obsolete
        # before) to be deleted. make sure to not delete any foreign
        # (that is, not from ubuntu) packages
        if self.useNetwork:
            # we can only do the obsoletes calculation here if we use a
            # network. otherwise after rewriting the sources.list everything
            # that is not on the CD becomes obsolete (not-downloadable)
            remove_candidates = now_obsolete - self.obsolete_pkgs
        else:
            # initial remove candidates when no network is used should
            # be the demotions to make sure we don't leave potential
            # unsupported software
            remove_candidates = set([p.name for p in self.installed_demotions])
        remove_candidates |= set(self.forced_obsoletes)

        # no go for the unused dependencies
        unused_dependencies = self.cache._getUnusedDependencies()
        logging.debug("Unused dependencies: %s" %" ".join(unused_dependencies))
        remove_candidates |= set(unused_dependencies)

        # see if we actually have to do anything here
        if not self.config.getWithDefault("Distro","RemoveObsoletes", True):
            logging.debug("Skipping RemoveObsoletes as stated in the config")
            remove_candidates = set()
        logging.debug("remove_candidates: '%s'" % remove_candidates)
        logging.debug("Start checking for obsolete pkgs")
        progress = self._view.getOpCacheProgress()
        for (i, pkgname) in enumerate(remove_candidates):
            progress.update((i/float(len(remove_candidates)))*100.0)
            if pkgname not in self.foreign_pkgs:
                self._view.processEvents()
                if not self.cache.tryMarkObsoleteForRemoval(pkgname, remove_candidates, self.foreign_pkgs):
                    logging.debug("'%s' scheduled for remove but not safe to remove, skipping", pkgname)
        logging.debug("Finish checking for obsolete pkgs")
        progress.done()

        # get changes
        changes = self.cache.get_changes()
        logging.debug("The following packages are marked for removal: %s" % " ".join([pkg.name for pkg in changes]))
        summary = _("Remove obsolete packages?")
        actions = [_("_Keep"), _("_Remove")]
        # FIXME Add an explanation about what obsolete packages are
        #explanation = _("")
        if (len(changes) > 0 and 
            self._view.confirmChanges(summary, changes, [], 0, actions, False)):
            fprogress = self._view.getAcquireProgress()
            iprogress = self._view.getInstallProgress(self.cache)
            try:
                self.cache.commit(fprogress,iprogress)
            except (SystemError, IOError) as e:
                logging.error("cache.commit() in doPostUpgrade() failed: %s" % e)
                self._view.error(_("Error during commit"),
                                 _("A problem occurred during the clean-up. "
                                   "Please see the below message for more "
                                   "information. "),
                                   "%s" % e)
        # run stuff after cleanup
        self.quirks.run("PostCleanup")
        # run the post upgrade scripts that can do fixup like xorg.conf
        # fixes etc - only do on real upgrades
        if not self._partialUpgrade:
            self.runPostInstallScripts()
        return True

    def runPostInstallScripts(self):
        """ 
        scripts that are run in any case after the distupgrade finished 
        whether or not it was successful
        """
        # now run the post-upgrade fixup scripts (if any)
        for script in self.config.getlist("Distro","PostInstallScripts"):
            if not os.path.exists(script):
                logging.warning("PostInstallScript: '%s' not found" % script)
                continue
            logging.debug("Running PostInstallScript: '%s'" % script)
            try:
                # work around kde tmpfile problem where it eats permissions
                check_and_fix_xbit(script)
                self._view.getTerminal().call([script], hidden=True)
            except Exception as e:
                logging.error("got error from PostInstallScript %s (%s)" % (script, e))
        
    def abort(self):
        """ abort the upgrade, cleanup (as much as possible) """
        logging.debug("abort called")
        if hasattr(self, "sources"):
            self.sources.restore_backup(self.sources_backup_ext)
        if hasattr(self, "aptcdrom"):
            self.aptcdrom.restore_backup(self.sources_backup_ext)
        # generate a new cache
        self._view.updateStatus(_("Restoring original system state"))
        self._view.abort()
        self.openCache()
        sys.exit(1)

    def _checkDep(self, depstr):
        " check if a given depends can be satisfied "
        for or_group in apt_pkg.parse_depends(depstr):
            logging.debug("checking: '%s' " % or_group)
            for dep in or_group:
                depname = dep[0]
                ver = dep[1]
                oper = dep[2]
                if depname not in self.cache:
                    logging.error("_checkDep: '%s' not in cache" % depname)
                    return False
                inst = self.cache[depname]
                instver = getattr(inst.installed, "version", None)
                if (instver != None and
                    apt_pkg.check_dep(instver,oper,ver) == True):
                    return True
        logging.error("depends '%s' is not satisfied" % depstr)
        return False
                
    def checkViewDepends(self):
        " check if depends are satisfied "
        logging.debug("checkViewDepends()")
        res = True
        # now check if anything from $foo-updates is required
        depends = self.config.getlist("View","Depends")
        depends.extend(self.config.getlist(self._view.__class__.__name__,
                                           "Depends"))
        for dep in depends:
            logging.debug("depends: '%s'", dep)
            res &= self._checkDep(dep)
            if not res:
                # FIXME: instead of error out, fetch and install it
                #        here
                self._view.error(_("Required depends is not installed"),
                                 _("The required dependency '%s' is not "
                                   "installed. " % dep))
                sys.exit(1)
        return res 

    def _verifyBackports(self):
        # run update (but ignore errors in case the countrymirror
        # substitution goes wrong, real errors will be caught later
        # when the cache is searched for the backport packages)
        backportslist = self.config.getlist("PreRequists","Packages")
        i=0
        noCache = apt_pkg.config.find("Acquire::http::No-Cache","false")
        maxRetries = self.config.getint("Network","MaxRetries")
        while i < maxRetries:
            self.doUpdate(showErrors=False)
            self.openCache()
            for pkgname in backportslist:
                if pkgname not in self.cache:
                    logging.error("Can not find backport '%s'" % pkgname)
                    raise NoBackportsFoundException(pkgname)
            if self._allBackportsAuthenticated(backportslist):
                break
            # FIXME: move this to some more generic place
            logging.debug("setting a cache control header to turn off caching temporarily")
            apt_pkg.config.set("Acquire::http::No-Cache","true")
            i += 1
        if i == maxRetries:
            logging.error("pre-requists item is NOT trusted, giving up")
            return False
        apt_pkg.config.set("Acquire::http::No-Cache",noCache)
        return True

    def _allBackportsAuthenticated(self, backportslist):
        # check if the user overwrote the check
        if apt_pkg.config.find_b("APT::Get::AllowUnauthenticated",False) == True:
            logging.warning("skip authentication check because of APT::Get::AllowUnauthenticated==true")
            return True
        try:
            b = self.config.getboolean("Distro","AllowUnauthenticated")
            if b:
                return True
        except NoOptionError:
            pass
        for pkgname in backportslist:
            pkg = self.cache[pkgname]
            if not pkg.candidate:
                return False
            for cand in pkg.candidate.origins:
                if cand.trusted:
                    break
            else:
                return False
        return True

    def isMirror(self, uri):
        """ check if uri is a known mirror """
        # deal with username:password in a netloc
        raw_uri = uri.rstrip("/")
        scheme, netloc, path, query, fragment = urlsplit(raw_uri)
        if "@" in netloc:
            netloc = netloc.split("@")[1]
        # construct new mirror url without the username/pw
        uri = "%s://%s%s" % (scheme, netloc, path)
        for mirror in self.valid_mirrors:
            mirror = mirror.rstrip("/")
            if is_mirror(mirror, uri):
                return True
            # deal with mirrors like
            #    deb http://localhost:9977/security.ubuntu.com/ubuntu intrepid-security main restricted
            # both apt-debtorrent and apt-cacher use this (LP: #365537)
            mirror_host_part = mirror.split("//")[1]
            if uri.endswith(mirror_host_part):
                logging.debug("found apt-cacher/apt-torrent style uri %s" % uri)
                return True
        return False

    def isThirdPartyMirror(self, uri):
        " check if uri is a whitelisted third-party mirror "
        uri = uri.rstrip("/")
        for mirror in self.valid_3p_mirrors:
            mirror = mirror.rstrip("/")
            if is_mirror(mirror, uri):
                return True
        return False

    def _getPreReqMirrorLines(self, dumb=False):
        " get sources.list snippet lines for the current mirror "
        lines = ""
        sources = SourcesList(matcherPath=".")
        for entry in sources.list:
            if entry.invalid or entry.disabled:
                continue
            if (entry.type == "deb" and 
                entry.disabled == False and
                self.isMirror(entry.uri) and
                "main" in entry.comps and
                "%s-updates" % self.fromDist in entry.dist and
                not entry.uri.startswith("http://security.ubuntu.com") and
                not entry.uri.startswith("http://archive.ubuntu.com") ):
                new_line = "deb %s %s-updates main\n" % (entry.uri, self.fromDist)
                if not new_line in lines:
                    lines += new_line
            # FIXME: do we really need "dumb" mode?
            #if (dumb and entry.type == "deb" and
            #    "main" in entry.comps):
            #    lines += "deb %s %s-proposed main\n" % (entry.uri, self.fromDist)
        return lines

    def _addPreRequistsSourcesList(self, template, out, dumb=False):
        " add prerequists based on template into the path outfile "
        # go over the sources.list and try to find a valid mirror
        # that we can use to add the backports dir
        logging.debug("writing prerequists sources.list at: '%s' " % out)
        outfile = open(out, "w")
        mirrorlines = self._getPreReqMirrorLines(dumb)
        for line in open(template):
            template = Template(line)
            outline = template.safe_substitute(mirror=mirrorlines)
            outfile.write(outline)
            logging.debug("adding '%s' prerequists" % outline)
        outfile.close()
        return True

    def getRequiredBackports(self):
        " download the backports specified in DistUpgrade.cfg "
        logging.debug("getRequiredBackports()")
        res = True
        backportsdir = os.path.join(os.getcwd(),"backports")
        if not os.path.exists(backportsdir):
            os.mkdir(backportsdir)
        backportslist = self.config.getlist("PreRequists","Packages")

        # FIXME: this needs to be ported
        # if we have them on the CD we are fine
        if self.aptcdrom and not self.useNetwork:
            logging.debug("Searching for pre-requists on CDROM")
            p = os.path.join(self.aptcdrom.cdrompath,
                             "dists/stable/main/dist-upgrader/binary-%s/" % apt_pkg.config.find("APT::Architecture"))
            found_pkgs = set()
            for deb in glob.glob(p+"*_*.deb"):
                logging.debug("found pre-req '%s' to '%s'" % (deb, backportsdir))
                found_pkgs.add(os.path.basename(deb).split("_")[0])
            # now check if we got all backports on the CD
            if not set(backportslist).issubset(found_pkgs):
                logging.error("Expected backports: '%s' but got '%s'" % (set(backportslist), found_pkgs))
                return False
            # now install them
            self.cache.release_lock()
            p = subprocess.Popen(
                ["/usr/bin/dpkg", "-i", ] + glob.glob(p+"*_*.deb"),
                stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                universal_newlines=True)
            res = None
            while res is None:
                res = p.poll()
                self._view.pulseProgress()
                time.sleep(0.02)
            self._view.pulseProgress(finished=True)
            self.cache.get_lock()
            logging.info("installing backport debs exit code '%s'" % res)
            logging.debug("dpkg output:\n%s" % p.communicate()[0])
            if res != 0:
                return False
            # and re-start itself when it done
            return self.setupRequiredBackports()

        # we support PreRequists/SourcesList-$arch sections here too
        # 
        # logic for mirror finding works list this:     
        # - use the mirror template from the config, then: [done]
        # 
        #  - try to find known mirror (isMirror) and prepend it [done]
        #  - archive.ubuntu.com is always a fallback at the end [done]
        # 
        # see if we find backports with that
        # - if not, try guessing based on URI, Trust and Dist   [done]
        #   in existing sources.list (internal mirror with no
        #   outside connection maybe)
        # 
        # make sure to remove file on cancel
        
        # FIXME: use the DistUpgradeFetcherCore logic
        #        in mirror_from_sources_list() here
        #        (and factor that code out into a helper)

        conf_option = "SourcesList"
        if self.config.has_option("PreRequists",conf_option+"-%s" % self.arch):
            conf_option = conf_option + "-%s" % self.arch
        prereq_template = self.config.get("PreRequists",conf_option)
        if not os.path.exists(prereq_template):
            logging.error("sourceslist not found '%s'" % prereq_template)
            return False
        outpath = os.path.join(apt_pkg.config.find_dir("Dir::Etc::sourceparts"), prereq_template)
        outfile = os.path.join(apt_pkg.config.find_dir("Dir::Etc::sourceparts"), prereq_template)
        self._addPreRequistsSourcesList(prereq_template, outfile) 
        try:
            self._verifyBackports()
        except NoBackportsFoundException as e:
            self._addPreRequistsSourcesList(prereq_template, outfile, dumb=True) 
            try:
                self._verifyBackports()
            except NoBackportsFoundException as e:
                logging.warning("no backport for '%s' found" % e)
            return False
        
        # FIXME: sanity check the origin (just for safety)
        for pkgname in backportslist:
            pkg = self.cache[pkgname]
            # look for the right version (backport)
            ver = self.cache._depcache.get_candidate_ver(pkg._pkg)
            if not ver:
                logging.error("No candidate for '%s'" % pkgname)
                os.unlink(outpath)
                return False
            if ver.file_list == None:
                logging.error("No ver.file_list for '%s'" % pkgname)
                os.unlink(outpath)
                return False
            logging.debug("marking '%s' for install" % pkgname)
            # mark install
            pkg.mark_install(auto_inst=False, auto_fix=False)

        # now get it
        res = False
        try:
            res = self.cache.commit(self._view.getAcquireProgress(),
                                    self._view.getInstallProgress(self.cache))
        except IOError as e:
            logging.error("fetch_archives returned '%s'" % e)
            res = False
        except SystemError as e:
            logging.error("install_archives returned '%s'" % e)
            res = False

        if res == False:
            logging.warning("_fetch_archives for backports returned False")

        # all backports done, remove the pre-requirests.list file again
        try:
            os.unlink(outfile)
        except Exception as e:
            logging.error("failed to unlink pre-requists file: '%s'" % e)
        return self.setupRequiredBackports()

    # used by both cdrom/http fetcher
    def setupRequiredBackports(self):
        # ensure that the new release upgrader uses the latest python-apt
        # from the backport path
        os.environ["PYTHONPATH"] = "/usr/lib/release-upgrader-python-apt"
        # copy log so that it gets not overwritten
        logging.shutdown()
        shutil.copy("/var/log/dist-upgrade/main.log",
                    "/var/log/dist-upgrade/main_pre_req.log")
        # now exec self again
        args = sys.argv + ["--have-prerequists"]
        if self.useNetwork:
            args.append("--with-network")
        else:
            args.append("--without-network")
        logging.info("restarting upgrader")
        #print("restarting upgrader to make use of the backports")
        # work around kde being clever and removing the x bit
        check_and_fix_xbit(sys.argv[0])
        os.execve(sys.argv[0],args, os.environ)

    # this is the core
    def fullUpgrade(self):
        # sanity check (check for ubuntu-desktop, brokenCache etc)
        self._view.updateStatus(_("Checking package manager"))
        self._view.setStep(STEP_PREPARE)

        if not self.prepare():
            logging.error("self.prepared() failed")
            if os.path.exists("/usr/bin/apport-bug"):
                self._view.error(_("Preparing the upgrade failed"),
                                 _("Preparing the system for the upgrade "
                                   "failed so a bug reporting process is "
                                   "being started."))
                subprocess.Popen(["apport-bug", "ubuntu-release-upgrader-core"])
            else:
                self._view.error(_("Preparing the upgrade failed"),
                                 _("Preparing the system for the upgrade "
                                   "failed. To report a bug install apport "
                                   "and then execute 'apport-bug "
                                   "ubuntu-release-upgrader'."))
                logging.error("Missing apport-bug, bug report not "
                              "autocreated")
            sys.exit(1)

        # mvo: commented out for now, see #54234, this needs to be
        #      refactored to use a arch=any tarball
        if (self.config.has_section("PreRequists") and
            self.options and
            self.options.havePrerequists == False):
            logging.debug("need backports")
            # get backported packages (if needed)
            if not self.getRequiredBackports():
                if os.path.exists("/usr/bin/apport-bug"):
                    self._view.error(_("Getting upgrade prerequisites failed"),
                                     _("The system was unable to get the "
                                       "prerequisites for the upgrade. "
                                       "The upgrade will abort now and restore "
                                       "the original system state.\n"
                                       "\n"
                                       "Additionally, a bug reporting process is "
                                       "being started."))
                    subprocess.Popen(["apport-bug", "ubuntu-release-upgrader-core"])
                else:
                    self._view.error(_("Getting upgrade prerequisites failed"),
                                     _("The system was unable to get the "
                                       "prerequisites for the upgrade. "
                                       "The upgrade will abort now and restore "
                                       "the original system state.\n"
                                       "\n"
                                       "To report a bug install apport and "
                                       "then execute 'apport-bug "
                                       "ubuntu-release-upgrader'."))
                    logging.error("Missing apport-bug, bug report not "
                                  "autocreated")
                self.abort()

        # run a "apt-get update" now, its ok to ignore errors, 
        # because 
        # a) we disable any third party sources later
        # b) we check if we have valid ubuntu sources later
        #    after we rewrite the sources.list and do a 
        #    apt-get update there too
        # because the (unmodified) sources.list of the user
        # may contain bad/unreachable entries we run only
        # with a single retry
        self.doUpdate(showErrors=False, forceRetries=1)
        self.openCache()

        # do pre-upgrade stuff (calc list of obsolete pkgs etc)
        if not self.doPostInitialUpdate():
            self.abort()

        # update sources.list
        self._view.setStep(STEP_MODIFY_SOURCES)
        self._view.updateStatus(_("Updating repository information"))
        if not self.updateSourcesList():
            self.abort()

        # add cdrom (if we have one)
        if (self.aptcdrom and
            not self.aptcdrom.add(self.sources_backup_ext)):
            self._view.error(_("Failed to add the cdrom"),
                             _("Sorry, adding the cdrom was not successful."))
            sys.exit(1)

        # then update the package index files
        if not self.doUpdate():
            self.abort()

        # then open the cache (again)
        self._view.updateStatus(_("Checking package manager"))
        # if something fails here (e.g. locking the cache) we need to
        # restore the system state (LP: #1052605)
        self.openCache(restore_sources_list_on_fail=True)

        # re-check server mode because we got new packages (it may happen
        # that the system had no sources.list entries and therefore no
        # desktop file information)
        self.serverMode = self.cache.need_server_mode()
        # do it here as we need to know if we are in server or client mode
        self.quirks.ensure_recommends_are_installed_on_desktops()
        # now check if we still have some key packages available/downloadable
        # after the update - if not something went seriously wrong
        # (this happend e.g. during the intrepid->jaunty upgrade for some
        #  users when de.archive.ubuntu.com was overloaded)
        for pkg in self.config.getlist("Distro","BaseMetaPkgs"):
            if (pkg not in self.cache or
                not self.cache.anyVersionDownloadable(self.cache[pkg])):
                # FIXME: we could offer to add default source entries here,
                #        but we need to be careful to not duplicate them
                #        (i.e. the error here could be something else than
                #        missing sources entries but network errors etc)
                logging.error("No '%s' available/downloadable after sources.list rewrite+update" % pkg)
                self._view.error(_("Invalid package information"),
                                 _("After updating your package "
                                   "information, the essential package '%s' "
                                   "could not be located. This may be "
                                   "because you have no official mirrors "
                                   "listed in your software sources, or "
                                   "because of excessive load on the mirror "
                                   "you are using. See /etc/apt/sources.list "
                                   "for the current list of configured "
                                   "software sources."
                                   "\n"
                                   "In the case of an overloaded mirror, you "
                                   "may want to try the upgrade again later.")
                                   % pkg)
                if os.path.exists("/usr/bin/apport-bug"):
                    subprocess.Popen(["apport-bug", "ubuntu-release-upgrader-core"])
                else:
                    logging.error("Missing apport-bug, bug report not "
                                  "autocreated")
                self.abort()

        # calc the dist-upgrade and see if the removals are ok/expected
        # do the dist-upgrade
        self._view.updateStatus(_("Calculating the changes"))
        if not self.askDistUpgrade():
            self.abort()

        # fetch the stuff
        self._view.setStep(STEP_FETCH)
        self._view.updateStatus(_("Fetching"))
        if not self.doDistUpgradeFetching():
            self.abort()

        # now do the upgrade
        self._view.setStep(STEP_INSTALL)
        self._view.updateStatus(_("Upgrading"))
        if not self.doDistUpgrade():
            # run the post install scripts (for stuff like UUID conversion)
            self.runPostInstallScripts()
            # don't abort here, because it would restore the sources.list
            self._view.information(_("Upgrade complete"),
                                   _("The upgrade has completed but there "
                                     "were errors during the upgrade "
                                     "process."))
            sys.exit(1) 
            
        # do post-upgrade stuff
        self._view.setStep(STEP_CLEANUP)
        self._view.updateStatus(_("Searching for obsolete software"))
        self.doPostUpgrade()

        # comment out cdrom source
        if self.aptcdrom:
            self.aptcdrom.comment_out_cdrom_entry()

        # remove upgrade-available notice
        if os.path.exists("/var/lib/ubuntu-release-upgrader/release-upgrade-available"):
            os.unlink("/var/lib/ubuntu-release-upgrader/release-upgrade-available")

        # done, ask for reboot
        self._view.setStep(STEP_REBOOT)
        self._view.updateStatus(_("System upgrade is complete."))            
        # FIXME should we look into /var/run/reboot-required here?
        if (not inside_chroot() and
            self._view.confirmRestart()):
            subprocess.Popen("/sbin/reboot")
            sys.exit(0)
        return True
        
    def run(self):
        self._view.processEvents()
        return self.fullUpgrade()
    
    def doPartialUpgrade(self):
        " partial upgrade mode, useful for repairing "
        self._view.setStep(STEP_PREPARE)
        self._view.hideStep(STEP_MODIFY_SOURCES)
        self._view.hideStep(STEP_REBOOT)
        self._partialUpgrade = True
        self.prepare()
        if not self.doPostInitialUpdate():
            return False
        if not self.askDistUpgrade():
            return False
        self._view.setStep(STEP_FETCH)
        self._view.updateStatus(_("Fetching"))
        if not self.doDistUpgradeFetching():
            return False
        self._view.setStep(STEP_INSTALL)
        self._view.updateStatus(_("Upgrading"))
        if not self.doDistUpgrade():
            self._view.information(_("Upgrade complete"),
                                   _("The upgrade has completed but there "
                                     "were errors during the upgrade "
                                     "process."))
            return False
        self._view.setStep(STEP_CLEANUP)
        if not self.doPostUpgrade():
            self._view.information(_("Upgrade complete"),
                                   _("The upgrade has completed but there "
                                     "were errors during the upgrade "
                                     "process."))
            return False

        if os.path.exists(REBOOT_REQUIRED_FILE):
            # we can not talk to session management here, we run as root
            if self._view.confirmRestart():
                subprocess.Popen("/sbin/reboot")
        else:
            self._view.information(_("Upgrade complete"),
                                   _("The partial upgrade was completed."))
        return True


if __name__ == "__main__":
    from .DistUpgradeViewText import DistUpgradeViewText
    logging.basicConfig(level=logging.DEBUG)
    v = DistUpgradeViewText()
    dc = DistUpgradeController(v)
    #dc.openCache()
    dc._disableAptCronJob()
    dc._enableAptCronJob()
    #dc._addRelatimeToFstab()
    #dc.prepare()
    #dc.askDistUpgrade()
    #dc._checkFreeSpace()
    #dc._rewriteFstab()
    #dc._checkAdminGroup()
    #dc._rewriteAptPeriodic(2)