/usr/lib/python2.7/dist-packages/bzrlib/transport/http/_urllib2_wrappers.py is in python-bzrlib 2.7.0-2ubuntu3.1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 | # Copyright (C) 2006-2013, 2016 Canonical Ltd
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
"""Implementation of urllib2 tailored to bzr needs
This file complements the urllib2 class hierarchy with custom classes.
For instance, we create a new HTTPConnection and HTTPSConnection that inherit
from the original urllib2.HTTP(s)Connection objects, but also have a new base
which implements a custom getresponse and cleanup_pipe handlers.
And then we implement custom HTTPHandler and HTTPSHandler classes, that use
the custom HTTPConnection classes.
We have a custom Response class, which lets us maintain a keep-alive
connection even for requests that urllib2 doesn't expect to contain body data.
And a custom Request class that lets us track redirections, and
handle authentication schemes.
For coherency with python libraries, we use capitalized header names throughout
the code, even if the header names will be titled just before sending the
request (see AbstractHTTPHandler.do_open).
"""
from __future__ import absolute_import
DEBUG = 0
# FIXME: Oversimplifying, two kind of exceptions should be
# raised, once a request is issued: URLError before we have been
# able to process the response, HTTPError after that. Process the
# response means we are able to leave the socket clean, so if we
# are not able to do that, we should close the connection. The
# actual code more or less do that, tests should be written to
# ensure that.
import base64
import errno
import httplib
import os
import socket
import urllib
import urllib2
import urlparse
import re
import ssl
import sys
import time
from bzrlib import __version__ as bzrlib_version
from bzrlib import (
config,
debug,
errors,
lazy_import,
osutils,
trace,
transport,
ui,
urlutils,
)
try:
_ = (ssl.match_hostname, ssl.CertificateError)
except AttributeError:
# Provide fallbacks for python < 2.7.9
def match_hostname(cert, host):
trace.warning(
'%s cannot be verified, https certificates verification is only'
' available for python versions >= 2.7.9' % (host,))
ssl.match_hostname = match_hostname
ssl.CertificateError = ValueError
# Note for packagers: if there is no package providing certs for your platform,
# the curl project produces http://curl.haxx.se/ca/cacert.pem weekly.
_ssl_ca_certs_known_locations = [
u'/etc/ssl/certs/ca-certificates.crt', # Ubuntu/debian/gentoo
u'/etc/pki/tls/certs/ca-bundle.crt', # Fedora/CentOS/RH
u'/etc/ssl/ca-bundle.pem', # OpenSuse
u'/etc/ssl/cert.pem', # OpenSuse
u"/usr/local/share/certs/ca-root-nss.crt", # FreeBSD
# XXX: Needs checking, can't trust the interweb ;) -- vila 2012-01-25
u'/etc/openssl/certs/ca-certificates.crt', # Solaris
]
def default_ca_certs():
if sys.platform == 'win32':
return os.path.join(os.path.dirname(sys.executable), u"cacert.pem")
elif sys.platform == 'darwin':
# FIXME: Needs some default value for osx, waiting for osx installers
# guys feedback -- vila 2012-01-25
pass
else:
# Try known locations for friendly OSes providing the root certificates
# without making them hard to use for any https client.
for path in _ssl_ca_certs_known_locations:
if os.path.exists(path):
# First found wins
return path
# A default path that makes sense and will be mentioned in the error
# presented to the user, even if not correct for all platforms
return _ssl_ca_certs_known_locations[0]
def ca_certs_from_store(path):
if not os.path.exists(path):
raise ValueError("ca certs path %s does not exist" % path)
return path
def cert_reqs_from_store(unicode_str):
import ssl
try:
return {"required": ssl.CERT_REQUIRED,
"none": ssl.CERT_NONE}[unicode_str]
except KeyError:
raise ValueError("invalid value %s" % unicode_str)
def default_ca_reqs():
if sys.platform in ('win32', 'darwin'):
# FIXME: Once we get a native access to root certificates there, this
# won't needed anymore. See http://pad.lv/920455 -- vila 2012-02-15
return u'none'
else:
return u'required'
opt_ssl_ca_certs = config.Option('ssl.ca_certs',
from_unicode=ca_certs_from_store,
default=default_ca_certs,
invalid='warning',
help="""\
Path to certification authority certificates to trust.
This should be a valid path to a bundle containing all root Certificate
Authorities used to verify an https server certificate.
Use ssl.cert_reqs=none to disable certificate verification.
""")
opt_ssl_cert_reqs = config.Option('ssl.cert_reqs',
default=default_ca_reqs,
from_unicode=cert_reqs_from_store,
invalid='error',
help="""\
Whether to require a certificate from the remote side. (default:required)
Possible values:
* none: Certificates ignored
* required: Certificates required and validated
""")
checked_kerberos = False
kerberos = None
class addinfourl(urllib2.addinfourl):
'''Replacement addinfourl class compatible with python-2.7's xmlrpclib
In python-2.7, xmlrpclib expects that the response object that it receives
has a getheader method. httplib.HTTPResponse provides this but
urllib2.addinfourl does not. Add the necessary functions here, ported to
use the internal data structures of addinfourl.
'''
def getheader(self, name, default=None):
if self.headers is None:
raise httplib.ResponseNotReady()
return self.headers.getheader(name, default)
def getheaders(self):
if self.headers is None:
raise httplib.ResponseNotReady()
return self.headers.items()
class _ReportingFileSocket(object):
def __init__(self, filesock, report_activity=None):
self.filesock = filesock
self._report_activity = report_activity
def report_activity(self, size, direction):
if self._report_activity:
self._report_activity(size, direction)
def read(self, size=1):
s = self.filesock.read(size)
self.report_activity(len(s), 'read')
return s
def readline(self, size=-1):
s = self.filesock.readline(size)
self.report_activity(len(s), 'read')
return s
def __getattr__(self, name):
return getattr(self.filesock, name)
class _ReportingSocket(object):
def __init__(self, sock, report_activity=None):
self.sock = sock
self._report_activity = report_activity
def report_activity(self, size, direction):
if self._report_activity:
self._report_activity(size, direction)
def sendall(self, s, *args):
self.sock.sendall(s, *args)
self.report_activity(len(s), 'write')
def recv(self, *args):
s = self.sock.recv(*args)
self.report_activity(len(s), 'read')
return s
def makefile(self, mode='r', bufsize=-1):
# httplib creates a fileobject that doesn't do buffering, which
# makes fp.readline() very expensive because it only reads one byte
# at a time. So we wrap the socket in an object that forces
# sock.makefile to make a buffered file.
fsock = self.sock.makefile(mode, 65536)
# And wrap that into a reporting kind of fileobject
return _ReportingFileSocket(fsock, self._report_activity)
def __getattr__(self, name):
return getattr(self.sock, name)
# We define our own Response class to keep our httplib pipe clean
class Response(httplib.HTTPResponse):
"""Custom HTTPResponse, to avoid the need to decorate.
httplib prefers to decorate the returned objects, rather
than using a custom object.
"""
# Some responses have bodies in which we have no interest
_body_ignored_responses = [301,302, 303, 307, 400, 401, 403, 404, 501]
# in finish() below, we may have to discard several MB in the worst
# case. To avoid buffering that much, we read and discard by chunks
# instead. The underlying file is either a socket or a StringIO, so reading
# 8k chunks should be fine.
_discarded_buf_size = 8192
def begin(self):
"""Begin to read the response from the server.
httplib assumes that some responses get no content and do
not even attempt to read the body in that case, leaving
the body in the socket, blocking the next request. Let's
try to workaround that.
"""
httplib.HTTPResponse.begin(self)
if self.status in self._body_ignored_responses:
if self.debuglevel >= 2:
print "For status: [%s]," % self.status,
print "will ready body, length: %s" % self.length
if not (self.length is None or self.will_close):
# In some cases, we just can't read the body not
# even try or we may encounter a 104, 'Connection
# reset by peer' error if there is indeed no body
# and the server closed the connection just after
# having issued the response headers (even if the
# headers indicate a Content-Type...)
body = self.read(self.length)
if self.debuglevel >= 9:
# This one can be huge and is generally not interesting
print "Consumed body: [%s]" % body
self.close()
elif self.status == 200:
# Whatever the request is, it went ok, so we surely don't want to
# close the connection. Some cases are not correctly detected by
# httplib.HTTPConnection.getresponse (called by
# httplib.HTTPResponse.begin). The CONNECT response for the https
# through proxy case is one. Note: the 'will_close' below refers
# to the "true" socket between us and the server, whereas the
# 'close()' above refers to the copy of that socket created by
# httplib for the response itself. So, in the if above we close the
# socket to indicate that we are done with the response whereas
# below we keep the socket with the server opened.
self.will_close = False
def finish(self):
"""Finish reading the body.
In some cases, the client may have left some bytes to read in the
body. That will block the next request to succeed if we use a
persistent connection. If we don't use a persistent connection, well,
nothing will block the next request since a new connection will be
issued anyway.
:return: the number of bytes left on the socket (may be None)
"""
pending = None
if not self.isclosed():
# Make sure nothing was left to be read on the socket
pending = 0
data = True
while data and self.length:
# read() will update self.length
data = self.read(min(self.length, self._discarded_buf_size))
pending += len(data)
if pending:
trace.mutter("%s bytes left on the HTTP socket", pending)
self.close()
return pending
# Not inheriting from 'object' because httplib.HTTPConnection doesn't.
class AbstractHTTPConnection:
"""A custom HTTP(S) Connection, which can reset itself on a bad response"""
response_class = Response
# When we detect a server responding with the whole file to range requests,
# we want to warn. But not below a given thresold.
_range_warning_thresold = 1024 * 1024
def __init__(self, report_activity=None):
self._response = None
self._report_activity = report_activity
self._ranges_received_whole_file = None
def _mutter_connect(self):
netloc = '%s:%s' % (self.host, self.port)
if self.proxied_host is not None:
netloc += '(proxy for %s)' % self.proxied_host
trace.mutter('* About to connect() to %s' % netloc)
def getresponse(self):
"""Capture the response to be able to cleanup"""
self._response = httplib.HTTPConnection.getresponse(self)
return self._response
def cleanup_pipe(self):
"""Read the remaining bytes of the last response if any."""
if self._response is not None:
try:
pending = self._response.finish()
# Warn the user (once)
if (self._ranges_received_whole_file is None
and self._response.status == 200
and pending and pending > self._range_warning_thresold
):
self._ranges_received_whole_file = True
trace.warning(
'Got a 200 response when asking for multiple ranges,'
' does your server at %s:%s support range requests?',
self.host, self.port)
except socket.error, e:
# It's conceivable that the socket is in a bad state here
# (including some test cases) and in this case, it doesn't need
# cleaning anymore, so no need to fail, we just get rid of the
# socket and let callers reconnect
if (len(e.args) == 0
or e.args[0] not in (errno.ECONNRESET, errno.ECONNABORTED)):
raise
self.close()
self._response = None
# Preserve our preciousss
sock = self.sock
self.sock = None
# Let httplib.HTTPConnection do its housekeeping
self.close()
# Restore our preciousss
self.sock = sock
def _wrap_socket_for_reporting(self, sock):
"""Wrap the socket before anybody use it."""
self.sock = _ReportingSocket(sock, self._report_activity)
class HTTPConnection(AbstractHTTPConnection, httplib.HTTPConnection):
# XXX: Needs refactoring at the caller level.
def __init__(self, host, port=None, proxied_host=None,
report_activity=None, ca_certs=None):
AbstractHTTPConnection.__init__(self, report_activity=report_activity)
# Use strict=True since we don't support HTTP/0.9
httplib.HTTPConnection.__init__(self, host, port, strict=True)
self.proxied_host = proxied_host
# ca_certs is ignored, it's only relevant for https
def connect(self):
if 'http' in debug.debug_flags:
self._mutter_connect()
httplib.HTTPConnection.connect(self)
self._wrap_socket_for_reporting(self.sock)
class HTTPSConnection(AbstractHTTPConnection, httplib.HTTPSConnection):
def __init__(self, host, port=None, key_file=None, cert_file=None,
proxied_host=None,
report_activity=None, ca_certs=None):
AbstractHTTPConnection.__init__(self, report_activity=report_activity)
# Use strict=True since we don't support HTTP/0.9
httplib.HTTPSConnection.__init__(self, host, port,
key_file, cert_file, strict=True)
self.proxied_host = proxied_host
self.ca_certs = ca_certs
def connect(self):
if 'http' in debug.debug_flags:
self._mutter_connect()
httplib.HTTPConnection.connect(self)
self._wrap_socket_for_reporting(self.sock)
if self.proxied_host is None:
self.connect_to_origin()
def connect_to_origin(self):
# FIXME JRV 2011-12-18: Use location config here?
config_stack = config.GlobalStack()
cert_reqs = config_stack.get('ssl.cert_reqs')
if self.proxied_host is not None:
host = self.proxied_host.split(":", 1)[0]
else:
host = self.host
if cert_reqs == ssl.CERT_NONE:
ui.ui_factory.show_user_warning('not_checking_ssl_cert', host=host)
ui.ui_factory.suppressed_warnings.add('not_checking_ssl_cert')
ca_certs = None
else:
if self.ca_certs is None:
ca_certs = config_stack.get('ssl.ca_certs')
else:
ca_certs = self.ca_certs
if ca_certs is None:
trace.warning(
"No valid trusted SSL CA certificates file set. See "
"'bzr help ssl.ca_certs' for more information on setting "
"trusted CAs.")
try:
ssl_sock = ssl.wrap_socket(
self.sock, self.key_file, self.cert_file,
cert_reqs=cert_reqs, ca_certs=ca_certs)
except ssl.SSLError:
trace.note(
"\n"
"See `bzr help ssl.ca_certs` for how to specify trusted CA"
"certificates.\n"
"Pass -Ossl.cert_reqs=none to disable certificate "
"verification entirely.\n")
raise
if cert_reqs == ssl.CERT_REQUIRED:
peer_cert = ssl_sock.getpeercert()
ssl.match_hostname(peer_cert, host)
# Wrap the ssl socket before anybody use it
self._wrap_socket_for_reporting(ssl_sock)
class Request(urllib2.Request):
"""A custom Request object.
urllib2 determines the request method heuristically (based on
the presence or absence of data). We set the method
statically.
The Request object tracks:
- the connection the request will be made on.
- the authentication parameters needed to preventively set
the authentication header once a first authentication have
been made.
"""
def __init__(self, method, url, data=None, headers={},
origin_req_host=None, unverifiable=False,
connection=None, parent=None,
accepted_errors=None):
urllib2.Request.__init__(self, url, data, headers,
origin_req_host, unverifiable)
self.method = method
self.connection = connection
self.accepted_errors = accepted_errors
# To handle redirections
self.parent = parent
self.redirected_to = None
# Unless told otherwise, redirections are not followed
self.follow_redirections = False
# auth and proxy_auth are dicts containing, at least
# (scheme, host, port, realm, user, password, protocol, path).
# The dict entries are mostly handled by the AuthHandler.
# Some authentication schemes may add more entries.
self.auth = {}
self.proxy_auth = {}
self.proxied_host = None
def get_method(self):
return self.method
def set_proxy(self, proxy, type):
"""Set the proxy and remember the proxied host."""
host, port = urllib.splitport(self.get_host())
if port is None:
# We need to set the default port ourselves way before it gets set
# in the HTTP[S]Connection object at build time.
if self.type == 'https':
conn_class = HTTPSConnection
else:
conn_class = HTTPConnection
port = conn_class.default_port
self.proxied_host = '%s:%s' % (host, port)
urllib2.Request.set_proxy(self, proxy, type)
# When urllib2 makes a https request with our wrapper code and a proxy,
# it sets Host to the https proxy, not the host we want to talk to.
# I'm fairly sure this is our fault, but what is the cause is an open
# question. -- Robert Collins May 8 2010.
self.add_unredirected_header('Host', self.proxied_host)
class _ConnectRequest(Request):
def __init__(self, request):
"""Constructor
:param request: the first request sent to the proxied host, already
processed by the opener (i.e. proxied_host is already set).
"""
# We give a fake url and redefine get_selector or urllib2 will be
# confused
Request.__init__(self, 'CONNECT', request.get_full_url(),
connection=request.connection)
if request.proxied_host is None:
raise AssertionError()
self.proxied_host = request.proxied_host
def get_selector(self):
return self.proxied_host
def set_proxy(self, proxy, type):
"""Set the proxy without remembering the proxied host.
We already know the proxied host by definition, the CONNECT request
occurs only when the connection goes through a proxy. The usual
processing (masquerade the request so that the connection is done to
the proxy while the request is targeted at another host) does not apply
here. In fact, the connection is already established with proxy and we
just want to enable the SSL tunneling.
"""
urllib2.Request.set_proxy(self, proxy, type)
class ConnectionHandler(urllib2.BaseHandler):
"""Provides connection-sharing by pre-processing requests.
urllib2 provides no way to access the HTTPConnection object
internally used. But we need it in order to achieve
connection sharing. So, we add it to the request just before
it is processed, and then we override the do_open method for
http[s] requests in AbstractHTTPHandler.
"""
handler_order = 1000 # after all pre-processings
def __init__(self, report_activity=None, ca_certs=None):
self._report_activity = report_activity
self.ca_certs = ca_certs
def create_connection(self, request, http_connection_class):
host = request.get_host()
if not host:
# Just a bit of paranoia here, this should have been
# handled in the higher levels
raise errors.InvalidURL(request.get_full_url(), 'no host given.')
# We create a connection (but it will not connect until the first
# request is made)
try:
connection = http_connection_class(
host, proxied_host=request.proxied_host,
report_activity=self._report_activity,
ca_certs=self.ca_certs)
except httplib.InvalidURL, exception:
# There is only one occurrence of InvalidURL in httplib
raise errors.InvalidURL(request.get_full_url(),
extra='nonnumeric port')
return connection
def capture_connection(self, request, http_connection_class):
"""Capture or inject the request connection.
Two cases:
- the request have no connection: create a new one,
- the request have a connection: this one have been used
already, let's capture it, so that we can give it to
another transport to be reused. We don't do that
ourselves: the Transport object get the connection from
a first request and then propagate it, from request to
request or to cloned transports.
"""
connection = request.connection
if connection is None:
# Create a new one
connection = self.create_connection(request, http_connection_class)
request.connection = connection
# All connections will pass here, propagate debug level
connection.set_debuglevel(DEBUG)
return request
def http_request(self, request):
return self.capture_connection(request, HTTPConnection)
def https_request(self, request):
return self.capture_connection(request, HTTPSConnection)
class AbstractHTTPHandler(urllib2.AbstractHTTPHandler):
"""A custom handler for HTTP(S) requests.
We overrive urllib2.AbstractHTTPHandler to get a better
control of the connection, the ability to implement new
request types and return a response able to cope with
persistent connections.
"""
# We change our order to be before urllib2 HTTP[S]Handlers
# and be chosen instead of them (the first http_open called
# wins).
handler_order = 400
_default_headers = {'Pragma': 'no-cache',
'Cache-control': 'max-age=0',
'Connection': 'Keep-Alive',
'User-agent': 'bzr/%s (urllib)' % bzrlib_version,
'Accept': '*/*',
}
def __init__(self):
urllib2.AbstractHTTPHandler.__init__(self, debuglevel=DEBUG)
def http_request(self, request):
"""Common headers setting"""
request.headers.update(self._default_headers.copy())
# FIXME: We may have to add the Content-Length header if
# we have data to send.
return request
def retry_or_raise(self, http_class, request, first_try):
"""Retry the request (once) or raise the exception.
urllib2 raises exception of application level kind, we
just have to translate them.
httplib can raise exceptions of transport level (badly
formatted dialog, loss of connexion or socket level
problems). In that case we should issue the request again
(httplib will close and reopen a new connection if
needed).
"""
# When an exception occurs, we give back the original
# Traceback or the bugs are hard to diagnose.
exc_type, exc_val, exc_tb = sys.exc_info()
if exc_type == socket.gaierror:
# No need to retry, that will not help
raise errors.ConnectionError("Couldn't resolve host '%s'"
% request.get_origin_req_host(),
orig_error=exc_val)
elif isinstance(exc_val, httplib.ImproperConnectionState):
# The httplib pipeline is in incorrect state, it's a bug in our
# implementation.
raise exc_type, exc_val, exc_tb
else:
if first_try:
if self._debuglevel >= 2:
print 'Received exception: [%r]' % exc_val
print ' On connection: [%r]' % request.connection
method = request.get_method()
url = request.get_full_url()
print ' Will retry, %s %r' % (method, url)
request.connection.close()
response = self.do_open(http_class, request, False)
else:
if self._debuglevel >= 2:
print 'Received second exception: [%r]' % exc_val
print ' On connection: [%r]' % request.connection
if exc_type in (httplib.BadStatusLine, httplib.UnknownProtocol):
# httplib.BadStatusLine and
# httplib.UnknownProtocol indicates that a
# bogus server was encountered or a bad
# connection (i.e. transient errors) is
# experimented, we have already retried once
# for that request so we raise the exception.
my_exception = errors.InvalidHttpResponse(
request.get_full_url(),
'Bad status line received',
orig_error=exc_val)
elif (isinstance(exc_val, socket.error) and len(exc_val.args)
and exc_val.args[0] in (errno.ECONNRESET, 10053, 10054)):
# 10053 == WSAECONNABORTED
# 10054 == WSAECONNRESET
raise errors.ConnectionReset(
"Connection lost while sending request.")
else:
# All other exception are considered connection related.
# socket errors generally occurs for reasons
# far outside our scope, so closing the
# connection and retrying is the best we can
# do.
my_exception = errors.ConnectionError(
msg= 'while sending %s %s:' % (request.get_method(),
request.get_selector()),
orig_error=exc_val)
if self._debuglevel >= 2:
print 'On connection: [%r]' % request.connection
method = request.get_method()
url = request.get_full_url()
print ' Failed again, %s %r' % (method, url)
print ' Will raise: [%r]' % my_exception
raise my_exception, None, exc_tb
return response
def do_open(self, http_class, request, first_try=True):
"""See urllib2.AbstractHTTPHandler.do_open for the general idea.
The request will be retried once if it fails.
"""
connection = request.connection
if connection is None:
raise AssertionError(
'Cannot process a request without a connection')
# Get all the headers
headers = {}
headers.update(request.header_items())
headers.update(request.unredirected_hdrs)
# Some servers or proxies will choke on headers not properly
# cased. httplib/urllib/urllib2 all use capitalize to get canonical
# header names, but only python2.5 urllib2 use title() to fix them just
# before sending the request. And not all versions of python 2.5 do
# that. Since we replace urllib2.AbstractHTTPHandler.do_open we do it
# ourself below.
headers = dict((name.title(), val) for name, val in headers.iteritems())
try:
method = request.get_method()
url = request.get_selector()
connection._send_request(method, url,
# FIXME: implements 100-continue
#None, # We don't send the body yet
request.get_data(),
headers)
if 'http' in debug.debug_flags:
trace.mutter('> %s %s' % (method, url))
hdrs = []
for k,v in headers.iteritems():
# People are often told to paste -Dhttp output to help
# debug. Don't compromise credentials.
if k in ('Authorization', 'Proxy-Authorization'):
v = '<masked>'
hdrs.append('%s: %s' % (k, v))
trace.mutter('> ' + '\n> '.join(hdrs) + '\n')
if self._debuglevel >= 1:
print 'Request sent: [%r] from (%s)' \
% (request, request.connection.sock.getsockname())
response = connection.getresponse()
convert_to_addinfourl = True
except (ssl.SSLError, ssl.CertificateError):
# Something is wrong with either the certificate or the hostname,
# re-trying won't help
raise
except (socket.gaierror, httplib.BadStatusLine, httplib.UnknownProtocol,
socket.error, httplib.HTTPException):
response = self.retry_or_raise(http_class, request, first_try)
convert_to_addinfourl = False
# FIXME: HTTPConnection does not fully support 100-continue (the
# server responses are just ignored)
# if code == 100:
# mutter('Will send the body')
# # We can send the body now
# body = request.get_data()
# if body is None:
# raise URLError("No data given")
# connection.send(body)
# response = connection.getresponse()
if self._debuglevel >= 2:
print 'Receives response: %r' % response
print ' For: %r(%r)' % (request.get_method(),
request.get_full_url())
if convert_to_addinfourl:
# Shamelessly copied from urllib2
req = request
r = response
r.recv = r.read
fp = socket._fileobject(r, bufsize=65536)
resp = addinfourl(fp, r.msg, req.get_full_url())
resp.code = r.status
resp.msg = r.reason
resp.version = r.version
if self._debuglevel >= 2:
print 'Create addinfourl: %r' % resp
print ' For: %r(%r)' % (request.get_method(),
request.get_full_url())
if 'http' in debug.debug_flags:
version = 'HTTP/%d.%d'
try:
version = version % (resp.version / 10,
resp.version % 10)
except:
version = 'HTTP/%r' % resp.version
trace.mutter('< %s %s %s' % (version, resp.code,
resp.msg))
# Use the raw header lines instead of treating resp.info() as a
# dict since we may miss duplicated headers otherwise.
hdrs = [h.rstrip('\r\n') for h in resp.info().headers]
trace.mutter('< ' + '\n< '.join(hdrs) + '\n')
else:
resp = response
return resp
class HTTPHandler(AbstractHTTPHandler):
"""A custom handler that just thunks into HTTPConnection"""
def http_open(self, request):
return self.do_open(HTTPConnection, request)
class HTTPSHandler(AbstractHTTPHandler):
"""A custom handler that just thunks into HTTPSConnection"""
https_request = AbstractHTTPHandler.http_request
def https_open(self, request):
connection = request.connection
if connection.sock is None and \
connection.proxied_host is not None and \
request.get_method() != 'CONNECT' : # Don't loop
# FIXME: We need a gazillion connection tests here, but we still
# miss a https server :-( :
# - with and without proxy
# - with and without certificate
# - with self-signed certificate
# - with and without authentication
# - with good and bad credentials (especially the proxy auth around
# CONNECT)
# - with basic and digest schemes
# - reconnection on errors
# - connection persistence behaviour (including reconnection)
# We are about to connect for the first time via a proxy, we must
# issue a CONNECT request first to establish the encrypted link
connect = _ConnectRequest(request)
response = self.parent.open(connect)
if response.code != 200:
raise errors.ConnectionError("Can't connect to %s via proxy %s" % (
connect.proxied_host, self.host))
# Housekeeping
connection.cleanup_pipe()
# Establish the connection encryption
connection.connect_to_origin()
# Propagate the connection to the original request
request.connection = connection
return self.do_open(HTTPSConnection, request)
class HTTPRedirectHandler(urllib2.HTTPRedirectHandler):
"""Handles redirect requests.
We have to implement our own scheme because we use a specific
Request object and because we want to implement a specific
policy.
"""
_debuglevel = DEBUG
# RFC2616 says that only read requests should be redirected
# without interacting with the user. But bzr use some
# shortcuts to optimize against roundtrips which can leads to
# write requests being issued before read requests of
# containing dirs can be redirected. So we redirect write
# requests in the same way which seems to respect the spirit
# of the RFC if not its letter.
def redirect_request(self, req, fp, code, msg, headers, newurl):
"""See urllib2.HTTPRedirectHandler.redirect_request"""
# We would have preferred to update the request instead
# of creating a new one, but the urllib2.Request object
# has a too complicated creation process to provide a
# simple enough equivalent update process. Instead, when
# redirecting, we only update the following request in
# the redirect chain with a reference to the parent
# request .
# Some codes make no sense in our context and are treated
# as errors:
# 300: Multiple choices for different representations of
# the URI. Using that mechanisn with bzr will violate the
# protocol neutrality of Transport.
# 304: Not modified (SHOULD only occurs with conditional
# GETs which are not used by our implementation)
# 305: Use proxy. I can't imagine this one occurring in
# our context-- vila/20060909
# 306: Unused (if the RFC says so...)
# If the code is 302 and the request is HEAD, some may
# think that it is a sufficent hint that the file exists
# and that we MAY avoid following the redirections. But
# if we want to be sure, we MUST follow them.
if code in (301, 302, 303, 307):
return Request(req.get_method(),newurl,
headers = req.headers,
origin_req_host = req.get_origin_req_host(),
unverifiable = True,
# TODO: It will be nice to be able to
# detect virtual hosts sharing the same
# IP address, that will allow us to
# share the same connection...
connection = None,
parent = req,
)
else:
raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
def http_error_302(self, req, fp, code, msg, headers):
"""Requests the redirected to URI.
Copied from urllib2 to be able to clean the pipe of the associated
connection, *before* issuing the redirected request but *after* having
eventually raised an error.
"""
# Some servers (incorrectly) return multiple Location headers
# (so probably same goes for URI). Use first header.
# TODO: Once we get rid of addinfourl objects, the
# following will need to be updated to use correct case
# for headers.
if 'location' in headers:
newurl = headers.getheaders('location')[0]
elif 'uri' in headers:
newurl = headers.getheaders('uri')[0]
else:
return
if self._debuglevel >= 1:
print 'Redirected to: %s (followed: %r)' % (newurl,
req.follow_redirections)
if req.follow_redirections is False:
req.redirected_to = newurl
return fp
newurl = urlparse.urljoin(req.get_full_url(), newurl)
# This call succeeds or raise an error. urllib2 returns
# if redirect_request returns None, but our
# redirect_request never returns None.
redirected_req = self.redirect_request(req, fp, code, msg, headers,
newurl)
# loop detection
# .redirect_dict has a key url if url was previously visited.
if hasattr(req, 'redirect_dict'):
visited = redirected_req.redirect_dict = req.redirect_dict
if (visited.get(newurl, 0) >= self.max_repeats or
len(visited) >= self.max_redirections):
raise urllib2.HTTPError(req.get_full_url(), code,
self.inf_msg + msg, headers, fp)
else:
visited = redirected_req.redirect_dict = req.redirect_dict = {}
visited[newurl] = visited.get(newurl, 0) + 1
# We can close the fp now that we are sure that we won't
# use it with HTTPError.
fp.close()
# We have all we need already in the response
req.connection.cleanup_pipe()
return self.parent.open(redirected_req)
http_error_301 = http_error_303 = http_error_307 = http_error_302
class ProxyHandler(urllib2.ProxyHandler):
"""Handles proxy setting.
Copied and modified from urllib2 to be able to modify the request during
the request pre-processing instead of modifying it at _open time. As we
capture (or create) the connection object during request processing, _open
time was too late.
The main task is to modify the request so that the connection is done to
the proxy while the request still refers to the destination host.
Note: the proxy handling *may* modify the protocol used; the request may be
against an https server proxied through an http proxy. So, https_request
will be called, but later it's really http_open that will be called. This
explains why we don't have to call self.parent.open as the urllib2 did.
"""
# Proxies must be in front
handler_order = 100
_debuglevel = DEBUG
def __init__(self, proxies=None):
urllib2.ProxyHandler.__init__(self, proxies)
# First, let's get rid of urllib2 implementation
for type, proxy in self.proxies.items():
if self._debuglevel >= 3:
print 'Will unbind %s_open for %r' % (type, proxy)
delattr(self, '%s_open' % type)
def bind_scheme_request(proxy, scheme):
if proxy is None:
return
scheme_request = scheme + '_request'
if self._debuglevel >= 3:
print 'Will bind %s for %r' % (scheme_request, proxy)
setattr(self, scheme_request,
lambda request: self.set_proxy(request, scheme))
# We are interested only by the http[s] proxies
http_proxy = self.get_proxy_env_var('http')
bind_scheme_request(http_proxy, 'http')
https_proxy = self.get_proxy_env_var('https')
bind_scheme_request(https_proxy, 'https')
def get_proxy_env_var(self, name, default_to='all'):
"""Get a proxy env var.
Note that we indirectly rely on
urllib.getproxies_environment taking into account the
uppercased values for proxy variables.
"""
try:
return self.proxies[name.lower()]
except KeyError:
if default_to is not None:
# Try to get the alternate environment variable
try:
return self.proxies[default_to]
except KeyError:
pass
return None
def proxy_bypass(self, host):
"""Check if host should be proxied or not.
:returns: True to skip the proxy, False otherwise.
"""
no_proxy = self.get_proxy_env_var('no', default_to=None)
bypass = self.evaluate_proxy_bypass(host, no_proxy)
if bypass is None:
# Nevertheless, there are platform-specific ways to
# ignore proxies...
return urllib.proxy_bypass(host)
else:
return bypass
def evaluate_proxy_bypass(self, host, no_proxy):
"""Check the host against a comma-separated no_proxy list as a string.
:param host: ``host:port`` being requested
:param no_proxy: comma-separated list of hosts to access directly.
:returns: True to skip the proxy, False not to, or None to
leave it to urllib.
"""
if no_proxy is None:
# All hosts are proxied
return False
hhost, hport = urllib.splitport(host)
# Does host match any of the domains mentioned in
# no_proxy ? The rules about what is authorized in no_proxy
# are fuzzy (to say the least). We try to allow most
# commonly seen values.
for domain in no_proxy.split(','):
domain = domain.strip()
if domain == '':
continue
dhost, dport = urllib.splitport(domain)
if hport == dport or dport is None:
# Protect glob chars
dhost = dhost.replace(".", r"\.")
dhost = dhost.replace("*", r".*")
dhost = dhost.replace("?", r".")
if re.match(dhost, hhost, re.IGNORECASE):
return True
# Nothing explicitly avoid the host
return None
def set_proxy(self, request, type):
if self.proxy_bypass(request.get_host()):
return request
proxy = self.get_proxy_env_var(type)
if self._debuglevel >= 3:
print 'set_proxy %s_request for %r' % (type, proxy)
# FIXME: python 2.5 urlparse provides a better _parse_proxy which can
# grok user:password@host:port as well as
# http://user:password@host:port
parsed_url = transport.ConnectedTransport._split_url(proxy)
if not parsed_url.host:
raise errors.InvalidURL(proxy, 'No host component')
if request.proxy_auth == {}:
# No proxy auth parameter are available, we are handling the first
# proxied request, intialize. scheme (the authentication scheme)
# and realm will be set by the AuthHandler
request.proxy_auth = {
'host': parsed_url.host,
'port': parsed_url.port,
'user': parsed_url.user,
'password': parsed_url.password,
'protocol': parsed_url.scheme,
# We ignore path since we connect to a proxy
'path': None}
if parsed_url.port is None:
phost = parsed_url.host
else:
phost = parsed_url.host + ':%d' % parsed_url.port
request.set_proxy(phost, type)
if self._debuglevel >= 3:
print 'set_proxy: proxy set to %s://%s' % (type, phost)
return request
class AbstractAuthHandler(urllib2.BaseHandler):
"""A custom abstract authentication handler for all http authentications.
Provides the meat to handle authentication errors and
preventively set authentication headers after the first
successful authentication.
This can be used for http and proxy, as well as for basic, negotiate and
digest authentications.
This provides an unified interface for all authentication handlers
(urllib2 provides far too many with different policies).
The interaction between this handler and the urllib2
framework is not obvious, it works as follow:
opener.open(request) is called:
- that may trigger http_request which will add an authentication header
(self.build_header) if enough info is available.
- the request is sent to the server,
- if an authentication error is received self.auth_required is called,
we acquire the authentication info in the error headers and call
self.auth_match to check that we are able to try the
authentication and complete the authentication parameters,
- we call parent.open(request), that may trigger http_request
and will add a header (self.build_header), but here we have
all the required info (keep in mind that the request and
authentication used in the recursive calls are really (and must be)
the *same* objects).
- if the call returns a response, the authentication have been
successful and the request authentication parameters have been updated.
"""
scheme = None
"""The scheme as it appears in the server header (lower cased)"""
_max_retry = 3
"""We don't want to retry authenticating endlessly"""
requires_username = True
"""Whether the auth mechanism requires a username."""
# The following attributes should be defined by daughter
# classes:
# - auth_required_header: the header received from the server
# - auth_header: the header sent in the request
def __init__(self):
# We want to know when we enter into an try/fail cycle of
# authentications so we initialize to None to indicate that we aren't
# in such a cycle by default.
self._retry_count = None
def _parse_auth_header(self, server_header):
"""Parse the authentication header.
:param server_header: The value of the header sent by the server
describing the authenticaion request.
:return: A tuple (scheme, remainder) scheme being the first word in the
given header (lower cased), remainder may be None.
"""
try:
scheme, remainder = server_header.split(None, 1)
except ValueError:
scheme = server_header
remainder = None
return (scheme.lower(), remainder)
def update_auth(self, auth, key, value):
"""Update a value in auth marking the auth as modified if needed"""
old_value = auth.get(key, None)
if old_value != value:
auth[key] = value
auth['modified'] = True
def auth_required(self, request, headers):
"""Retry the request if the auth scheme is ours.
:param request: The request needing authentication.
:param headers: The headers for the authentication error response.
:return: None or the response for the authenticated request.
"""
# Don't try to authenticate endlessly
if self._retry_count is None:
# The retry being recusrsive calls, None identify the first retry
self._retry_count = 1
else:
self._retry_count += 1
if self._retry_count > self._max_retry:
# Let's be ready for next round
self._retry_count = None
return None
server_headers = headers.getheaders(self.auth_required_header)
if not server_headers:
# The http error MUST have the associated
# header. This must never happen in production code.
raise KeyError('%s not found' % self.auth_required_header)
auth = self.get_auth(request)
auth['modified'] = False
# Put some common info in auth if the caller didn't
if auth.get('path', None) is None:
parsed_url = urlutils.URL.from_string(request.get_full_url())
self.update_auth(auth, 'protocol', parsed_url.scheme)
self.update_auth(auth, 'host', parsed_url.host)
self.update_auth(auth, 'port', parsed_url.port)
self.update_auth(auth, 'path', parsed_url.path)
# FIXME: the auth handler should be selected at a single place instead
# of letting all handlers try to match all headers, but the current
# design doesn't allow a simple implementation.
for server_header in server_headers:
# Several schemes can be proposed by the server, try to match each
# one in turn
matching_handler = self.auth_match(server_header, auth)
if matching_handler:
# auth_match may have modified auth (by adding the
# password or changing the realm, for example)
if (request.get_header(self.auth_header, None) is not None
and not auth['modified']):
# We already tried that, give up
return None
# Only the most secure scheme proposed by the server should be
# used, since the handlers use 'handler_order' to describe that
# property, the first handler tried takes precedence, the
# others should not attempt to authenticate if the best one
# failed.
best_scheme = auth.get('best_scheme', None)
if best_scheme is None:
# At that point, if current handler should doesn't succeed
# the credentials are wrong (or incomplete), but we know
# that the associated scheme should be used.
best_scheme = auth['best_scheme'] = self.scheme
if best_scheme != self.scheme:
continue
if self.requires_username and auth.get('user', None) is None:
# Without a known user, we can't authenticate
return None
# Housekeeping
request.connection.cleanup_pipe()
# Retry the request with an authentication header added
response = self.parent.open(request)
if response:
self.auth_successful(request, response)
return response
# We are not qualified to handle the authentication.
# Note: the authentication error handling will try all
# available handlers. If one of them authenticates
# successfully, a response will be returned. If none of
# them succeeds, None will be returned and the error
# handler will raise the 401 'Unauthorized' or the 407
# 'Proxy Authentication Required' error.
return None
def add_auth_header(self, request, header):
"""Add the authentication header to the request"""
request.add_unredirected_header(self.auth_header, header)
def auth_match(self, header, auth):
"""Check that we are able to handle that authentication scheme.
The request authentication parameters may need to be
updated with info from the server. Some of these
parameters, when combined, are considered to be the
authentication key, if one of them change the
authentication result may change. 'user' and 'password'
are exampls, but some auth schemes may have others
(digest's nonce is an example, digest's nonce_count is a
*counter-example*). Such parameters must be updated by
using the update_auth() method.
:param header: The authentication header sent by the server.
:param auth: The auth parameters already known. They may be
updated.
:returns: True if we can try to handle the authentication.
"""
raise NotImplementedError(self.auth_match)
def build_auth_header(self, auth, request):
"""Build the value of the header used to authenticate.
:param auth: The auth parameters needed to build the header.
:param request: The request needing authentication.
:return: None or header.
"""
raise NotImplementedError(self.build_auth_header)
def auth_successful(self, request, response):
"""The authentification was successful for the request.
Additional infos may be available in the response.
:param request: The succesfully authenticated request.
:param response: The server response (may contain auth info).
"""
# It may happen that we need to reconnect later, let's be ready
self._retry_count = None
def get_user_password(self, auth):
"""Ask user for a password if none is already available.
:param auth: authentication info gathered so far (from the initial url
and then during dialog with the server).
"""
auth_conf = config.AuthenticationConfig()
user = auth.get('user', None)
password = auth.get('password', None)
realm = auth['realm']
port = auth.get('port', None)
if user is None:
user = auth_conf.get_user(auth['protocol'], auth['host'],
port=port, path=auth['path'],
realm=realm, ask=True,
prompt=self.build_username_prompt(auth))
if user is not None and password is None:
password = auth_conf.get_password(
auth['protocol'], auth['host'], user,
port=port,
path=auth['path'], realm=realm,
prompt=self.build_password_prompt(auth))
return user, password
def _build_password_prompt(self, auth):
"""Build a prompt taking the protocol used into account.
The AuthHandler is used by http and https, we want that information in
the prompt, so we build the prompt from the authentication dict which
contains all the needed parts.
Also, http and proxy AuthHandlers present different prompts to the
user. The daughter classes should implements a public
build_password_prompt using this method.
"""
prompt = u'%s' % auth['protocol'].upper() + u' %(user)s@%(host)s'
realm = auth['realm']
if realm is not None:
prompt += u", Realm: '%s'" % realm.decode('utf8')
prompt += u' password'
return prompt
def _build_username_prompt(self, auth):
"""Build a prompt taking the protocol used into account.
The AuthHandler is used by http and https, we want that information in
the prompt, so we build the prompt from the authentication dict which
contains all the needed parts.
Also, http and proxy AuthHandlers present different prompts to the
user. The daughter classes should implements a public
build_username_prompt using this method.
"""
prompt = u'%s' % auth['protocol'].upper() + u' %(host)s'
realm = auth['realm']
if realm is not None:
prompt += u", Realm: '%s'" % realm.decode('utf8')
prompt += u' username'
return prompt
def http_request(self, request):
"""Insert an authentication header if information is available"""
auth = self.get_auth(request)
if self.auth_params_reusable(auth):
self.add_auth_header(request, self.build_auth_header(auth, request))
return request
https_request = http_request # FIXME: Need test
class NegotiateAuthHandler(AbstractAuthHandler):
"""A authentication handler that handles WWW-Authenticate: Negotiate.
At the moment this handler supports just Kerberos. In the future,
NTLM support may also be added.
"""
scheme = 'negotiate'
handler_order = 480
requires_username = False
def auth_match(self, header, auth):
scheme, raw_auth = self._parse_auth_header(header)
if scheme != self.scheme:
return False
self.update_auth(auth, 'scheme', scheme)
resp = self._auth_match_kerberos(auth)
if resp is None:
return False
# Optionally should try to authenticate using NTLM here
self.update_auth(auth, 'negotiate_response', resp)
return True
def _auth_match_kerberos(self, auth):
"""Try to create a GSSAPI response for authenticating against a host."""
global kerberos, checked_kerberos
if kerberos is None and not checked_kerberos:
try:
import kerberos
except ImportError:
kerberos = None
checked_kerberos = True
if kerberos is None:
return None
ret, vc = kerberos.authGSSClientInit("HTTP@%(host)s" % auth)
if ret < 1:
trace.warning('Unable to create GSSAPI context for %s: %d',
auth['host'], ret)
return None
ret = kerberos.authGSSClientStep(vc, "")
if ret < 0:
trace.mutter('authGSSClientStep failed: %d', ret)
return None
return kerberos.authGSSClientResponse(vc)
def build_auth_header(self, auth, request):
return "Negotiate %s" % auth['negotiate_response']
def auth_params_reusable(self, auth):
# If the auth scheme is known, it means a previous
# authentication was successful, all information is
# available, no further checks are needed.
return (auth.get('scheme', None) == 'negotiate' and
auth.get('negotiate_response', None) is not None)
class BasicAuthHandler(AbstractAuthHandler):
"""A custom basic authentication handler."""
scheme = 'basic'
handler_order = 500
auth_regexp = re.compile('realm="([^"]*)"', re.I)
def build_auth_header(self, auth, request):
raw = '%s:%s' % (auth['user'], auth['password'])
auth_header = 'Basic ' + base64.b64encode(raw)
return auth_header
def extract_realm(self, header_value):
match = self.auth_regexp.search(header_value)
realm = None
if match:
realm = match.group(1)
return match, realm
def auth_match(self, header, auth):
scheme, raw_auth = self._parse_auth_header(header)
if scheme != self.scheme:
return False
match, realm = self.extract_realm(raw_auth)
if match:
# Put useful info into auth
self.update_auth(auth, 'scheme', scheme)
self.update_auth(auth, 'realm', realm)
if (auth.get('user', None) is None
or auth.get('password', None) is None):
user, password = self.get_user_password(auth)
self.update_auth(auth, 'user', user)
self.update_auth(auth, 'password', password)
return match is not None
def auth_params_reusable(self, auth):
# If the auth scheme is known, it means a previous
# authentication was successful, all information is
# available, no further checks are needed.
return auth.get('scheme', None) == 'basic'
def get_digest_algorithm_impls(algorithm):
H = None
KD = None
if algorithm == 'MD5':
H = lambda x: osutils.md5(x).hexdigest()
elif algorithm == 'SHA':
H = osutils.sha_string
if H is not None:
KD = lambda secret, data: H("%s:%s" % (secret, data))
return H, KD
def get_new_cnonce(nonce, nonce_count):
raw = '%s:%d:%s:%s' % (nonce, nonce_count, time.ctime(),
urllib2.randombytes(8))
return osutils.sha_string(raw)[:16]
class DigestAuthHandler(AbstractAuthHandler):
"""A custom digest authentication handler."""
scheme = 'digest'
# Before basic as digest is a bit more secure and should be preferred
handler_order = 490
def auth_params_reusable(self, auth):
# If the auth scheme is known, it means a previous
# authentication was successful, all information is
# available, no further checks are needed.
return auth.get('scheme', None) == 'digest'
def auth_match(self, header, auth):
scheme, raw_auth = self._parse_auth_header(header)
if scheme != self.scheme:
return False
# Put the requested authentication info into a dict
req_auth = urllib2.parse_keqv_list(urllib2.parse_http_list(raw_auth))
# Check that we can handle that authentication
qop = req_auth.get('qop', None)
if qop != 'auth': # No auth-int so far
return False
H, KD = get_digest_algorithm_impls(req_auth.get('algorithm', 'MD5'))
if H is None:
return False
realm = req_auth.get('realm', None)
# Put useful info into auth
self.update_auth(auth, 'scheme', scheme)
self.update_auth(auth, 'realm', realm)
if auth.get('user', None) is None or auth.get('password', None) is None:
user, password = self.get_user_password(auth)
self.update_auth(auth, 'user', user)
self.update_auth(auth, 'password', password)
try:
if req_auth.get('algorithm', None) is not None:
self.update_auth(auth, 'algorithm', req_auth.get('algorithm'))
nonce = req_auth['nonce']
if auth.get('nonce', None) != nonce:
# A new nonce, never used
self.update_auth(auth, 'nonce_count', 0)
self.update_auth(auth, 'nonce', nonce)
self.update_auth(auth, 'qop', qop)
auth['opaque'] = req_auth.get('opaque', None)
except KeyError:
# Some required field is not there
return False
return True
def build_auth_header(self, auth, request):
url_scheme, url_selector = urllib.splittype(request.get_selector())
sel_host, uri = urllib.splithost(url_selector)
A1 = '%s:%s:%s' % (auth['user'], auth['realm'], auth['password'])
A2 = '%s:%s' % (request.get_method(), uri)
nonce = auth['nonce']
qop = auth['qop']
nonce_count = auth['nonce_count'] + 1
ncvalue = '%08x' % nonce_count
cnonce = get_new_cnonce(nonce, nonce_count)
H, KD = get_digest_algorithm_impls(auth.get('algorithm', 'MD5'))
nonce_data = '%s:%s:%s:%s:%s' % (nonce, ncvalue, cnonce, qop, H(A2))
request_digest = KD(H(A1), nonce_data)
header = 'Digest '
header += 'username="%s", realm="%s", nonce="%s"' % (auth['user'],
auth['realm'],
nonce)
header += ', uri="%s"' % uri
header += ', cnonce="%s", nc=%s' % (cnonce, ncvalue)
header += ', qop="%s"' % qop
header += ', response="%s"' % request_digest
# Append the optional fields
opaque = auth.get('opaque', None)
if opaque:
header += ', opaque="%s"' % opaque
if auth.get('algorithm', None):
header += ', algorithm="%s"' % auth.get('algorithm')
# We have used the nonce once more, update the count
auth['nonce_count'] = nonce_count
return header
class HTTPAuthHandler(AbstractAuthHandler):
"""Custom http authentication handler.
Send the authentication preventively to avoid the roundtrip
associated with the 401 error and keep the revelant info in
the auth request attribute.
"""
auth_required_header = 'www-authenticate'
auth_header = 'Authorization'
def get_auth(self, request):
"""Get the auth params from the request"""
return request.auth
def set_auth(self, request, auth):
"""Set the auth params for the request"""
request.auth = auth
def build_password_prompt(self, auth):
return self._build_password_prompt(auth)
def build_username_prompt(self, auth):
return self._build_username_prompt(auth)
def http_error_401(self, req, fp, code, msg, headers):
return self.auth_required(req, headers)
class ProxyAuthHandler(AbstractAuthHandler):
"""Custom proxy authentication handler.
Send the authentication preventively to avoid the roundtrip
associated with the 407 error and keep the revelant info in
the proxy_auth request attribute..
"""
auth_required_header = 'proxy-authenticate'
# FIXME: the correct capitalization is Proxy-Authorization,
# but python-2.4 urllib2.Request insist on using capitalize()
# instead of title().
auth_header = 'Proxy-authorization'
def get_auth(self, request):
"""Get the auth params from the request"""
return request.proxy_auth
def set_auth(self, request, auth):
"""Set the auth params for the request"""
request.proxy_auth = auth
def build_password_prompt(self, auth):
prompt = self._build_password_prompt(auth)
prompt = u'Proxy ' + prompt
return prompt
def build_username_prompt(self, auth):
prompt = self._build_username_prompt(auth)
prompt = u'Proxy ' + prompt
return prompt
def http_error_407(self, req, fp, code, msg, headers):
return self.auth_required(req, headers)
class HTTPBasicAuthHandler(BasicAuthHandler, HTTPAuthHandler):
"""Custom http basic authentication handler"""
class ProxyBasicAuthHandler(BasicAuthHandler, ProxyAuthHandler):
"""Custom proxy basic authentication handler"""
class HTTPDigestAuthHandler(DigestAuthHandler, HTTPAuthHandler):
"""Custom http basic authentication handler"""
class ProxyDigestAuthHandler(DigestAuthHandler, ProxyAuthHandler):
"""Custom proxy basic authentication handler"""
class HTTPNegotiateAuthHandler(NegotiateAuthHandler, HTTPAuthHandler):
"""Custom http negotiate authentication handler"""
class ProxyNegotiateAuthHandler(NegotiateAuthHandler, ProxyAuthHandler):
"""Custom proxy negotiate authentication handler"""
class HTTPErrorProcessor(urllib2.HTTPErrorProcessor):
"""Process HTTP error responses.
We don't really process the errors, quite the contrary
instead, we leave our Transport handle them.
"""
accepted_errors = [200, # Ok
206, # Partial content
404, # Not found
]
"""The error codes the caller will handle.
This can be specialized in the request on a case-by case basis, but the
common cases are covered here.
"""
def http_response(self, request, response):
code, msg, hdrs = response.code, response.msg, response.info()
accepted_errors = request.accepted_errors
if accepted_errors is None:
accepted_errors = self.accepted_errors
if code not in accepted_errors:
response = self.parent.error('http', request, response,
code, msg, hdrs)
return response
https_response = http_response
class HTTPDefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
"""Translate common errors into bzr Exceptions"""
def http_error_default(self, req, fp, code, msg, hdrs):
if code == 403:
raise errors.TransportError(
'Server refuses to fulfill the request (403 Forbidden)'
' for %s' % req.get_full_url())
else:
raise errors.InvalidHttpResponse(req.get_full_url(),
'Unable to handle http code %d: %s'
% (code, msg))
class Opener(object):
"""A wrapper around urllib2.build_opener
Daughter classes can override to build their own specific opener
"""
# TODO: Provides hooks for daughter classes.
def __init__(self,
connection=ConnectionHandler,
redirect=HTTPRedirectHandler,
error=HTTPErrorProcessor,
report_activity=None,
ca_certs=None):
self._opener = urllib2.build_opener(
connection(report_activity=report_activity, ca_certs=ca_certs),
redirect, error,
ProxyHandler(),
HTTPBasicAuthHandler(),
HTTPDigestAuthHandler(),
HTTPNegotiateAuthHandler(),
ProxyBasicAuthHandler(),
ProxyDigestAuthHandler(),
ProxyNegotiateAuthHandler(),
HTTPHandler,
HTTPSHandler,
HTTPDefaultErrorHandler,
)
self.open = self._opener.open
if DEBUG >= 9:
# When dealing with handler order, it's easy to mess
# things up, the following will help understand which
# handler is used, when and for what.
import pprint
pprint.pprint(self._opener.__dict__)
|