/usr/share/doc/python-requests-toolbelt-doc/html/threading.html is in python-requests-toolbelt-doc 0.7.0-1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 | <!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Using requests with Threading — requests_toolbelt 0.7.0 documentation</title>
<link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
<link rel="top" title="requests_toolbelt 0.7.0 documentation" href="index.html"/>
<link rel="next" title="Uploading Data" href="uploading-data.html"/>
<link rel="prev" title="Specialized Sessions" href="sessions.html"/>
<script src="_static/js/modernizr.min.js"></script>
</head>
<body class="wy-body-for-nav" role="document">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search">
<a href="index.html" class="icon icon-home"> requests_toolbelt
</a>
<div class="version">
0.7.0
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<ul>
<li class="toctree-l1"><a class="reference internal" href="user.html">requests toolbelt</a></li>
<li class="toctree-l1"><a class="reference internal" href="contributing.html">Contributing to this project</a></li>
</ul>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="adapters.html">Transport Adapters</a></li>
<li class="toctree-l1"><a class="reference internal" href="authentication.html">Authentication</a></li>
<li class="toctree-l1"><a class="reference internal" href="deprecated.html">Deprecated Requests Utilities</a></li>
<li class="toctree-l1"><a class="reference internal" href="downloadutils.html">Utilities for Downloading Streaming Responses</a></li>
<li class="toctree-l1"><a class="reference internal" href="dumputils.html">Utilities for Dumping Information About Responses</a></li>
<li class="toctree-l1"><a class="reference internal" href="formdata.html">Utilities for Enhanced Form-Data Serialization</a></li>
<li class="toctree-l1"><a class="reference internal" href="exceptions.html">Custom Toolbelt Exceptions</a></li>
<li class="toctree-l1"><a class="reference internal" href="sessions.html">Specialized Sessions</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">Using requests with Threading</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#module-requests_toolbelt.threaded">API and Module Auto-Generated Documentation</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="uploading-data.html">Uploading Data</a></li>
<li class="toctree-l1"><a class="reference internal" href="user-agent.html">User-Agent Constructor</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">requests_toolbelt</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="index.html">Docs</a> »</li>
<li>Using requests with Threading</li>
<li class="wy-breadcrumbs-aside">
<a href="_sources/threading.txt" rel="nofollow"> View page source</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<div class="section" id="using-requests-with-threading">
<span id="threading"></span><h1>Using requests with Threading<a class="headerlink" href="#using-requests-with-threading" title="Permalink to this headline">¶</a></h1>
<div class="versionadded">
<p><span class="versionmodified">New in version 0.4.0.</span></p>
</div>
<p>The toolbelt provides a simple API for using requests with threading.</p>
<p>A requests Session is documented as threadsafe but there are still a couple
corner cases where it isn’t perfectly threadsafe. The best way to use a
Session is to use one per thread.</p>
<p>The implementation provided by the toolbelt is naïve. This means that we use
one session per thread and we make no effort to synchronize attributes (e.g.,
authentication, cookies, etc.). It also means that we make no attempt to
direct a request to a session that has already handled a request to the same
domain. In other words, if you’re making requests to multiple domains, the
toolbelt’s Pool will not try to send requests to the same domain to the same
thread.</p>
<p>This module provides three classes:</p>
<ul class="simple">
<li><a class="reference internal" href="#requests_toolbelt.threaded.pool.Pool" title="requests_toolbelt.threaded.pool.Pool"><code class="xref py py-class docutils literal"><span class="pre">Pool</span></code></a></li>
<li><a class="reference internal" href="#requests_toolbelt.threaded.pool.ThreadResponse" title="requests_toolbelt.threaded.pool.ThreadResponse"><code class="xref py py-class docutils literal"><span class="pre">ThreadResponse</span></code></a></li>
<li><a class="reference internal" href="#requests_toolbelt.threaded.pool.ThreadException" title="requests_toolbelt.threaded.pool.ThreadException"><code class="xref py py-class docutils literal"><span class="pre">ThreadException</span></code></a></li>
</ul>
<p>In 98% of the situations you’ll want to just use a
<a class="reference internal" href="#requests_toolbelt.threaded.pool.Pool" title="requests_toolbelt.threaded.pool.Pool"><code class="xref py py-class docutils literal"><span class="pre">Pool</span></code></a> and you’ll treat a
<a class="reference internal" href="#requests_toolbelt.threaded.pool.ThreadResponse" title="requests_toolbelt.threaded.pool.ThreadResponse"><code class="xref py py-class docutils literal"><span class="pre">ThreadResponse</span></code></a> as if it were a
regular <code class="xref py py-class docutils literal"><span class="pre">requests.Response</span></code>.</p>
<p>Here’s an example:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="c1"># This example assumes Python 3</span>
<span class="kn">import</span> <span class="nn">queue</span>
<span class="kn">from</span> <span class="nn">requests_toolbelt.threaded</span> <span class="kn">import</span> <span class="n">pool</span>
<span class="n">jobs</span> <span class="o">=</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span><span class="p">()</span>
<span class="n">urls</span> <span class="o">=</span> <span class="p">[</span>
<span class="c1"># My list of URLs to get</span>
<span class="p">]</span>
<span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">urls</span><span class="p">:</span>
<span class="n">queue</span><span class="o">.</span><span class="n">put</span><span class="p">({</span><span class="s1">'method'</span><span class="p">:</span> <span class="s1">'GET'</span><span class="p">,</span> <span class="s1">'url'</span><span class="p">:</span> <span class="n">url</span><span class="p">})</span>
<span class="n">p</span> <span class="o">=</span> <span class="n">pool</span><span class="o">.</span><span class="n">Pool</span><span class="p">(</span><span class="n">job_queue</span><span class="o">=</span><span class="n">q</span><span class="p">)</span>
<span class="n">p</span><span class="o">.</span><span class="n">join_all</span><span class="p">()</span>
<span class="k">for</span> <span class="n">response</span> <span class="ow">in</span> <span class="n">p</span><span class="o">.</span><span class="n">responses</span><span class="p">():</span>
<span class="k">print</span><span class="p">(</span><span class="s1">'GET {0}. Returned {1}.'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">request_kwargs</span><span class="p">[</span><span class="s1">'url'</span><span class="p">],</span>
<span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="p">))</span>
</pre></div>
</div>
<p>This is clearly a bit underwhelming. This is why there’s a short-cut class
method to create a <a class="reference internal" href="#requests_toolbelt.threaded.pool.Pool" title="requests_toolbelt.threaded.pool.Pool"><code class="xref py py-class docutils literal"><span class="pre">Pool</span></code></a> from a list
of URLs.</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">requests_toolbelt.threaded</span> <span class="kn">import</span> <span class="n">pool</span>
<span class="n">urls</span> <span class="o">=</span> <span class="p">[</span>
<span class="c1"># My list of URLs to get</span>
<span class="p">]</span>
<span class="n">p</span> <span class="o">=</span> <span class="n">pool</span><span class="o">.</span><span class="n">Pool</span><span class="o">.</span><span class="n">from_urls</span><span class="p">(</span><span class="n">urls</span><span class="p">)</span>
<span class="n">p</span><span class="o">.</span><span class="n">join_all</span><span class="p">()</span>
<span class="k">for</span> <span class="n">response</span> <span class="ow">in</span> <span class="n">p</span><span class="o">.</span><span class="n">responses</span><span class="p">():</span>
<span class="k">print</span><span class="p">(</span><span class="s1">'GET {0}. Returned {1}.'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">request_kwargs</span><span class="p">[</span><span class="s1">'url'</span><span class="p">],</span>
<span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="p">))</span>
</pre></div>
</div>
<p>If one of the URLs in your list throws an exception, it will be accessible
from the <code class="xref py py-meth docutils literal"><span class="pre">exceptions()</span></code> generator.</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">requests_toolbelt.threaded</span> <span class="kn">import</span> <span class="n">pool</span>
<span class="n">urls</span> <span class="o">=</span> <span class="p">[</span>
<span class="c1"># My list of URLs to get</span>
<span class="p">]</span>
<span class="n">p</span> <span class="o">=</span> <span class="n">pool</span><span class="o">.</span><span class="n">Pool</span><span class="o">.</span><span class="n">from_urls</span><span class="p">(</span><span class="n">urls</span><span class="p">)</span>
<span class="n">p</span><span class="o">.</span><span class="n">join_all</span><span class="p">()</span>
<span class="k">for</span> <span class="n">exc</span> <span class="ow">in</span> <span class="n">p</span><span class="o">.</span><span class="n">exceptions</span><span class="p">():</span>
<span class="k">print</span><span class="p">(</span><span class="s1">'GET {0}. Raised {1}.'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">exc</span><span class="o">.</span><span class="n">request_kwargs</span><span class="p">[</span><span class="s1">'url'</span><span class="p">],</span>
<span class="n">exc</span><span class="o">.</span><span class="n">message</span><span class="p">))</span>
</pre></div>
</div>
<p>If instead, you want to retry the exceptions that have been raised you can do
the following:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">requests_toolbelt.threaded</span> <span class="kn">import</span> <span class="n">pool</span>
<span class="n">urls</span> <span class="o">=</span> <span class="p">[</span>
<span class="c1"># My list of URLs to get</span>
<span class="p">]</span>
<span class="n">p</span> <span class="o">=</span> <span class="n">pool</span><span class="o">.</span><span class="n">Pool</span><span class="o">.</span><span class="n">from_urls</span><span class="p">(</span><span class="n">urls</span><span class="p">)</span>
<span class="n">p</span><span class="o">.</span><span class="n">join_all</span><span class="p">()</span>
<span class="n">new_pool</span> <span class="o">=</span> <span class="n">pool</span><span class="o">.</span><span class="n">Pool</span><span class="o">.</span><span class="n">from_exceptions</span><span class="p">(</span><span class="n">p</span><span class="o">.</span><span class="n">exceptions</span><span class="p">())</span>
<span class="n">new_pool</span><span class="o">.</span><span class="n">join_all</span><span class="p">()</span>
</pre></div>
</div>
<p>Not all requests are advisable to retry without checking if they should be
retried. You would normally check if you want to retry it.</p>
<p>The <code class="xref py py-class docutils literal"><span class="pre">Pool</span></code> object takes 4 other keyword arguments:</p>
<ul>
<li><p class="first"><code class="docutils literal"><span class="pre">initializer</span></code></p>
<p>This is a callback that will initialize things on every session created. The
callback must return the session.</p>
</li>
<li><p class="first"><code class="docutils literal"><span class="pre">auth_generator</span></code></p>
<p>This is a callback that is called <em>after</em> the initializer callback has
modified the session. This callback must also return the session.</p>
</li>
<li><p class="first"><code class="docutils literal"><span class="pre">num_processes</span></code></p>
<p>By passing a positive integer that indicates how many threads to use. It is
<code class="docutils literal"><span class="pre">None</span></code> by default, and will use the result of
<code class="docutils literal"><span class="pre">multiproccessing.cpu_count()</span></code>.</p>
</li>
<li><p class="first"><code class="docutils literal"><span class="pre">session</span></code></p>
<p>You can pass an alternative constructor or any callable that returns a
<code class="xref py py-class docutils literal"><span class="pre">requests.Sesssion</span></code> like object. It will not be passed any arguments
because a <code class="xref py py-class docutils literal"><span class="pre">requests.Session</span></code> does not accept any arguments.</p>
</li>
</ul>
<p>Finally, if you don’t want to worry about Queue or Pool management, you can
try the following:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">requests_toolbelt</span> <span class="kn">import</span> <span class="n">threaded</span>
<span class="n">requests</span> <span class="o">=</span> <span class="p">[{</span>
<span class="s1">'method'</span><span class="p">:</span> <span class="s1">'GET'</span><span class="p">,</span>
<span class="s1">'url'</span><span class="p">:</span> <span class="s1">'https://httpbin.org/get'</span><span class="p">,</span>
<span class="c1"># ...</span>
<span class="p">},</span> <span class="p">{</span>
<span class="c1"># ...</span>
<span class="p">},</span> <span class="p">{</span>
<span class="c1"># ...</span>
<span class="p">}]</span>
<span class="n">responses_generator</span><span class="p">,</span> <span class="n">exceptions_generator</span> <span class="o">=</span> <span class="n">threaded</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">requests</span><span class="p">)</span>
<span class="k">for</span> <span class="n">response</span> <span class="ow">in</span> <span class="n">responses_generator</span><span class="p">:</span>
<span class="c1"># Do something</span>
</pre></div>
</div>
<div class="section" id="module-requests_toolbelt.threaded">
<span id="api-and-module-auto-generated-documentation"></span><h2>API and Module Auto-Generated Documentation<a class="headerlink" href="#module-requests_toolbelt.threaded" title="Permalink to this headline">¶</a></h2>
<p>This module provides the API for <code class="docutils literal"><span class="pre">requests_toolbelt.threaded</span></code>.</p>
<p>The module provides a clean and simple API for making requests via a thread
pool. The thread pool will use sessions for increased performance.</p>
<p>A simple use-case is:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">requests_toolbelt</span> <span class="kn">import</span> <span class="n">threaded</span>
<span class="n">urls_to_get</span> <span class="o">=</span> <span class="p">[{</span>
<span class="s1">'url'</span><span class="p">:</span> <span class="s1">'https://api.github.com/users/sigmavirus24'</span><span class="p">,</span>
<span class="s1">'method'</span><span class="p">:</span> <span class="s1">'GET'</span><span class="p">,</span>
<span class="p">},</span> <span class="p">{</span>
<span class="s1">'url'</span><span class="p">:</span> <span class="s1">'https://api.github.com/repos/sigmavirus24/requests-toolbelt'</span><span class="p">,</span>
<span class="s1">'method'</span><span class="p">:</span> <span class="s1">'GET'</span><span class="p">,</span>
<span class="p">},</span> <span class="p">{</span>
<span class="s1">'url'</span><span class="p">:</span> <span class="s1">'https://google.com'</span><span class="p">,</span>
<span class="s1">'method'</span><span class="p">:</span> <span class="s1">'GET'</span><span class="p">,</span>
<span class="p">}]</span>
<span class="n">responses</span><span class="p">,</span> <span class="n">errors</span> <span class="o">=</span> <span class="n">threaded</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">urls_to_get</span><span class="p">)</span>
</pre></div>
</div>
<p>By default, the threaded submodule will detect the number of CPUs your
computer has and use that if no other number of processes is selected. To
change this, always use the keyword argument <code class="docutils literal"><span class="pre">num_processes</span></code>. Using the
above example, we would expand it like so:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="n">responses</span><span class="p">,</span> <span class="n">errors</span> <span class="o">=</span> <span class="n">threaded</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">urls_to_get</span><span class="p">,</span> <span class="n">num_processes</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
</pre></div>
</div>
<p>You can also customize how a <code class="xref py py-class docutils literal"><span class="pre">requests.Session</span></code> is initialized by
creating a callback function:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">requests_toolbelt</span> <span class="kn">import</span> <span class="n">user_agent</span>
<span class="k">def</span> <span class="nf">initialize_session</span><span class="p">(</span><span class="n">session</span><span class="p">):</span>
<span class="n">session</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">'User-Agent'</span><span class="p">]</span> <span class="o">=</span> <span class="n">user_agent</span><span class="p">(</span><span class="s1">'my-scraper'</span><span class="p">,</span> <span class="s1">'0.1'</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">'Accept'</span><span class="p">]</span> <span class="o">=</span> <span class="s1">'application/json'</span>
<span class="n">responses</span><span class="p">,</span> <span class="n">errors</span> <span class="o">=</span> <span class="n">threaded</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">urls_to_get</span><span class="p">,</span>
<span class="n">initializer</span><span class="o">=</span><span class="n">initialize_session</span><span class="p">)</span>
</pre></div>
</div>
<dl class="function">
<dt id="requests_toolbelt.threaded.map">
<code class="descclassname">requests_toolbelt.threaded.</code><code class="descname">map</code><span class="sig-paren">(</span><em>requests</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="headerlink" href="#requests_toolbelt.threaded.map" title="Permalink to this definition">¶</a></dt>
<dd><p>Simple interface to the threaded Pool object.</p>
<p>This function takes a list of dictionaries representing requests to make
using Sessions in threads and returns a tuple where the first item is
a generator of successful responses and the second is a generator of
exceptions.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>requests</strong> (<a class="reference external" href="/usr/share/doc/python-doc/html/library/functions.html#list" title="(in Python v2.7)"><em>list</em></a>) – Collection of dictionaries representing requests to make with the Pool
object.</li>
<li><strong>**kwargs</strong> – Keyword arguments that are passed to the
<a class="reference internal" href="#requests_toolbelt.threaded.pool.Pool" title="requests_toolbelt.threaded.pool.Pool"><code class="xref py py-class docutils literal"><span class="pre">Pool</span></code></a> object.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">Tuple of responses and exceptions from the pool</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">(<a class="reference internal" href="#requests_toolbelt.threaded.pool.ThreadResponse" title="requests_toolbelt.threaded.pool.ThreadResponse"><code class="xref py py-class docutils literal"><span class="pre">ThreadResponse</span></code></a>,
<a class="reference internal" href="#requests_toolbelt.threaded.pool.ThreadException" title="requests_toolbelt.threaded.pool.ThreadException"><code class="xref py py-class docutils literal"><span class="pre">ThreadException</span></code></a>)</p>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<p>Inspiration is blatantly drawn from the standard library’s multiprocessing
library. See the following references:</p>
<ul class="simple">
<li>multiprocessing’s <a class="reference external" href="https://hg.python.org/cpython/file/8ef4f75a8018/Lib/multiprocessing/pool.py">pool source</a></li>
<li>map and map_async <a class="reference external" href="https://hg.python.org/cpython/file/8ef4f75a8018/Lib/multiprocessing/pool.py#l340">inspiration</a></li>
</ul>
<dl class="class">
<dt id="requests_toolbelt.threaded.pool.Pool">
<em class="property">class </em><code class="descclassname">requests_toolbelt.threaded.pool.</code><code class="descname">Pool</code><span class="sig-paren">(</span><em>job_queue</em>, <em>initializer=None</em>, <em>auth_generator=None</em>, <em>num_processes=None</em>, <em>session=<class 'requests.sessions.Session'></em><span class="sig-paren">)</span><a class="headerlink" href="#requests_toolbelt.threaded.pool.Pool" title="Permalink to this definition">¶</a></dt>
<dd><p>Pool that manages the threads containing sessions.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
<li><strong>queue</strong> (<em>queue.Queue</em>) – The queue you’re expected to use to which you should add items.</li>
<li><strong>initializer</strong> (<a class="reference external" href="/usr/share/doc/python-doc/html/library/collections.html#collections.Callable" title="(in Python v2.7)"><em>collections.Callable</em></a>) – Function used to initialize an instance of <code class="docutils literal"><span class="pre">session</span></code>.</li>
<li><strong>auth_generator</strong> (<a class="reference external" href="/usr/share/doc/python-doc/html/library/collections.html#collections.Callable" title="(in Python v2.7)"><em>collections.Callable</em></a>) – Function used to generate new auth credentials for the session.</li>
<li><strong>num_threads</strong> (<a class="reference external" href="/usr/share/doc/python-doc/html/library/functions.html#int" title="(in Python v2.7)"><em>int</em></a>) – Number of threads to create.</li>
<li><strong>session</strong> (<em>requests.Session</em>) – </li>
</ul>
</td>
</tr>
</tbody>
</table>
<dl class="method">
<dt id="requests_toolbelt.threaded.pool.Pool.exceptions">
<code class="descname">exceptions</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#requests_toolbelt.threaded.pool.Pool.exceptions" title="Permalink to this definition">¶</a></dt>
<dd><p>Iterate over all the exceptions in the pool.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Returns:</th><td class="field-body">Generator of <code class="xref py py-class docutils literal"><span class="pre">ThreadException</span></code></td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="classmethod">
<dt id="requests_toolbelt.threaded.pool.Pool.from_exceptions">
<em class="property">classmethod </em><code class="descname">from_exceptions</code><span class="sig-paren">(</span><em>exceptions</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="headerlink" href="#requests_toolbelt.threaded.pool.Pool.from_exceptions" title="Permalink to this definition">¶</a></dt>
<dd><p>Create a <code class="xref py py-class docutils literal"><span class="pre">Pool</span></code> from an <code class="xref py py-class docutils literal"><span class="pre">ThreadException</span></code>s.</p>
<p>Provided an iterable that provides <code class="xref py py-class docutils literal"><span class="pre">ThreadException</span></code> objects,
this classmethod will generate a new pool to retry the requests that
caused the exceptions.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>exceptions</strong> (<em>iterable</em>) – Iterable that returns <code class="xref py py-class docutils literal"><span class="pre">ThreadException</span></code></li>
<li><strong>kwargs</strong> – Keyword arguments passed to the <code class="xref py py-class docutils literal"><span class="pre">Pool</span></code> initializer.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">An initialized <code class="xref py py-class docutils literal"><span class="pre">Pool</span></code> object.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last"><code class="xref py py-class docutils literal"><span class="pre">Pool</span></code></p>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="classmethod">
<dt id="requests_toolbelt.threaded.pool.Pool.from_urls">
<em class="property">classmethod </em><code class="descname">from_urls</code><span class="sig-paren">(</span><em>urls</em>, <em>request_kwargs=None</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="headerlink" href="#requests_toolbelt.threaded.pool.Pool.from_urls" title="Permalink to this definition">¶</a></dt>
<dd><p>Create a <code class="xref py py-class docutils literal"><span class="pre">Pool</span></code> from an iterable of URLs.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>urls</strong> (<em>iterable</em>) – Iterable that returns URLs with which we create a pool.</li>
<li><strong>request_kwargs</strong> (<a class="reference external" href="/usr/share/doc/python-doc/html/library/stdtypes.html#dict" title="(in Python v2.7)"><em>dict</em></a>) – Dictionary of other keyword arguments to provide to the request
method.</li>
<li><strong>kwargs</strong> – Keyword arguments passed to the <code class="xref py py-class docutils literal"><span class="pre">Pool</span></code> initializer.</li>
</ul>
</td>
</tr>
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">An initialized <code class="xref py py-class docutils literal"><span class="pre">Pool</span></code> object.</p>
</td>
</tr>
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last"><code class="xref py py-class docutils literal"><span class="pre">Pool</span></code></p>
</td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="requests_toolbelt.threaded.pool.Pool.get_exception">
<code class="descname">get_exception</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#requests_toolbelt.threaded.pool.Pool.get_exception" title="Permalink to this definition">¶</a></dt>
<dd><p>Get an exception from the pool.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><code class="xref py py-class docutils literal"><span class="pre">ThreadException</span></code></td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="requests_toolbelt.threaded.pool.Pool.get_response">
<code class="descname">get_response</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#requests_toolbelt.threaded.pool.Pool.get_response" title="Permalink to this definition">¶</a></dt>
<dd><p>Get a response from the pool.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><code class="xref py py-class docutils literal"><span class="pre">ThreadResponse</span></code></td>
</tr>
</tbody>
</table>
</dd></dl>
<dl class="method">
<dt id="requests_toolbelt.threaded.pool.Pool.join_all">
<code class="descname">join_all</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#requests_toolbelt.threaded.pool.Pool.join_all" title="Permalink to this definition">¶</a></dt>
<dd><p>Join all the threads to the master thread.</p>
</dd></dl>
<dl class="method">
<dt id="requests_toolbelt.threaded.pool.Pool.responses">
<code class="descname">responses</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#requests_toolbelt.threaded.pool.Pool.responses" title="Permalink to this definition">¶</a></dt>
<dd><p>Iterate over all the responses in the pool.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
<tbody valign="top">
<tr class="field-odd field"><th class="field-name">Returns:</th><td class="field-body">Generator of <code class="xref py py-class docutils literal"><span class="pre">ThreadResponse</span></code></td>
</tr>
</tbody>
</table>
</dd></dl>
</dd></dl>
<dl class="class">
<dt id="requests_toolbelt.threaded.pool.ThreadResponse">
<em class="property">class </em><code class="descclassname">requests_toolbelt.threaded.pool.</code><code class="descname">ThreadResponse</code><span class="sig-paren">(</span><em>request_kwargs</em>, <em>response</em><span class="sig-paren">)</span><a class="headerlink" href="#requests_toolbelt.threaded.pool.ThreadResponse" title="Permalink to this definition">¶</a></dt>
<dd><p>A wrapper around a requests Response object.</p>
<p>This will proxy most attribute access actions to the Response object. For
example, if you wanted the parsed JSON from the response, you might do:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="n">thread_response</span> <span class="o">=</span> <span class="n">pool</span><span class="o">.</span><span class="n">get_response</span><span class="p">()</span>
<span class="n">json</span> <span class="o">=</span> <span class="n">thread_response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
</pre></div>
</div>
<dl class="attribute">
<dt id="requests_toolbelt.threaded.pool.ThreadResponse.request_kwargs">
<code class="descname">request_kwargs</code><em class="property"> = None</em><a class="headerlink" href="#requests_toolbelt.threaded.pool.ThreadResponse.request_kwargs" title="Permalink to this definition">¶</a></dt>
<dd><p>The original keyword arguments provided to the queue</p>
</dd></dl>
<dl class="attribute">
<dt id="requests_toolbelt.threaded.pool.ThreadResponse.response">
<code class="descname">response</code><em class="property"> = None</em><a class="headerlink" href="#requests_toolbelt.threaded.pool.ThreadResponse.response" title="Permalink to this definition">¶</a></dt>
<dd><p>The wrapped response</p>
</dd></dl>
</dd></dl>
<dl class="class">
<dt id="requests_toolbelt.threaded.pool.ThreadException">
<em class="property">class </em><code class="descclassname">requests_toolbelt.threaded.pool.</code><code class="descname">ThreadException</code><span class="sig-paren">(</span><em>request_kwargs</em>, <em>exception</em><span class="sig-paren">)</span><a class="headerlink" href="#requests_toolbelt.threaded.pool.ThreadException" title="Permalink to this definition">¶</a></dt>
<dd><p>A wrapper around an exception raised during a request.</p>
<p>This will proxy most attribute access actions to the exception object. For
example, if you wanted the message from the exception, you might do:</p>
<div class="highlight-python"><div class="highlight"><pre><span></span><span class="n">thread_exc</span> <span class="o">=</span> <span class="n">pool</span><span class="o">.</span><span class="n">get_exception</span><span class="p">()</span>
<span class="n">msg</span> <span class="o">=</span> <span class="n">thread_exc</span><span class="o">.</span><span class="n">message</span>
</pre></div>
</div>
<dl class="attribute">
<dt id="requests_toolbelt.threaded.pool.ThreadException.exception">
<code class="descname">exception</code><em class="property"> = None</em><a class="headerlink" href="#requests_toolbelt.threaded.pool.ThreadException.exception" title="Permalink to this definition">¶</a></dt>
<dd><p>The captured and wrapped exception</p>
</dd></dl>
<dl class="attribute">
<dt id="requests_toolbelt.threaded.pool.ThreadException.request_kwargs">
<code class="descname">request_kwargs</code><em class="property"> = None</em><a class="headerlink" href="#requests_toolbelt.threaded.pool.ThreadException.request_kwargs" title="Permalink to this definition">¶</a></dt>
<dd><p>The original keyword arguments provided to the queue</p>
</dd></dl>
</dd></dl>
</div>
</div>
</div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="uploading-data.html" class="btn btn-neutral float-right" title="Uploading Data" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a>
<a href="sessions.html" class="btn btn-neutral" title="Specialized Sessions" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
<hr/>
<div role="contentinfo">
<p>
© Copyright 2016, Ian Cordasco, Cory Benfield.
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'0.7.0',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true
};
</script>
<script type="text/javascript" src="_static/jquery.js"></script>
<script type="text/javascript" src="_static/underscore.js"></script>
<script type="text/javascript" src="_static/doctools.js"></script>
<script type="text/javascript" src="_static/js/theme.js"></script>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.StickyNav.enable();
});
</script>
</body>
</html>
|