python-dask-doc 0.16.0-1 / usr / share / doc / python-dask-doc / html / bag-overview.html

This file is indexed.

/usr/share/doc/python-dask-doc/html/bag-overview.html is in python-dask-doc 0.16.0-1.

This file is owned by root:root, with mode 0o644.

The actual contents of the file can be viewed below. 

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Overview &mdash; dask 0.16.0 documentation</title>
  

  
  
  
  

  

  
  
    

  

  
  
    <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
  

  
    <link rel="stylesheet" href="_static/style.css" type="text/css" />
  

  
        <link rel="index" title="Index"
              href="genindex.html"/>
        <link rel="search" title="Search" href="search.html"/>
    <link rel="top" title="dask 0.16.0 documentation" href="index.html"/>
        <link rel="up" title="Bag" href="bag.html"/>
        <link rel="next" title="Create Dask Bags" href="bag-creation.html"/>
        <link rel="prev" title="Bag" href="bag.html"/> 

  
  <script src="_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav" role="document">

   
  <div class="wy-grid-for-nav">

    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search">
          

          
            <a href="index.html" class="icon icon-home"> dask
          

          
          </a>

          
            
            
              <div class="version">
                0.16.0
              </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">Getting Started</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="install.html">Install Dask</a></li>
<li class="toctree-l1"><a class="reference internal" href="use-cases.html">Use Cases</a></li>
<li class="toctree-l1"><a class="reference internal" href="examples-tutorials.html">Examples</a></li>
<li class="toctree-l1"><a class="reference internal" href="cheatsheet.html">Dask Cheat Sheet</a></li>
</ul>
<p class="caption"><span class="caption-text">Collections</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="array.html">Array</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="bag.html">Bag</a><ul class="current">
<li class="toctree-l2 current"><a class="current reference internal" href="#">Overview</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#design">Design</a></li>
<li class="toctree-l3"><a class="reference internal" href="#common-uses">Common Uses</a></li>
<li class="toctree-l3"><a class="reference internal" href="#execution">Execution</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#default-scheduler">Default scheduler</a></li>
<li class="toctree-l4"><a class="reference internal" href="#shuffle">Shuffle</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="#known-limitations">Known Limitations</a></li>
<li class="toctree-l3"><a class="reference internal" href="#name">Name</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="bag-creation.html">Create Dask Bags</a></li>
<li class="toctree-l2"><a class="reference internal" href="bag-creation.html#store-dask-bags">Store Dask Bags</a></li>
<li class="toctree-l2"><a class="reference internal" href="bag-api.html">API</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="dataframe.html">DataFrame</a></li>
<li class="toctree-l1"><a class="reference internal" href="delayed.html">Delayed</a></li>
<li class="toctree-l1"><a class="reference internal" href="futures.html">Futures</a></li>
<li class="toctree-l1"><a class="reference internal" href="machine-learning.html">Machine Learning</a></li>
</ul>
<p class="caption"><span class="caption-text">Scheduling</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="distributed.html">Distributed Scheduling</a></li>
<li class="toctree-l1"><a class="reference internal" href="scheduler-overview.html">Scheduler Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="scheduler-choice.html">Choosing between Schedulers</a></li>
<li class="toctree-l1"><a class="reference internal" href="shared.html">Shared Memory</a></li>
<li class="toctree-l1"><a class="reference internal" href="scheduling-policy.html">Scheduling in Depth</a></li>
</ul>
<p class="caption"><span class="caption-text">Diagnostics</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="inspect.html">Inspecting Dask objects</a></li>
<li class="toctree-l1"><a class="reference internal" href="diagnostics.html">Diagnostics</a></li>
</ul>
<p class="caption"><span class="caption-text">Graphs</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="graphs.html">Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="spec.html">Specification</a></li>
<li class="toctree-l1"><a class="reference internal" href="custom-graphs.html">Custom Graphs</a></li>
<li class="toctree-l1"><a class="reference internal" href="optimize.html">Optimization</a></li>
</ul>
<p class="caption"><span class="caption-text">Help &amp; reference</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="debugging.html">Debugging</a></li>
<li class="toctree-l1"><a class="reference internal" href="support.html">Contact and Support</a></li>
<li class="toctree-l1"><a class="reference internal" href="changelog.html">Changelog</a></li>
<li class="toctree-l1"><a class="reference internal" href="presentations.html">Presentations On Dask</a></li>
<li class="toctree-l1"><a class="reference internal" href="develop.html">Development Guidelines</a></li>
<li class="toctree-l1"><a class="reference internal" href="faq.html">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="spark.html">Comparison to PySpark</a></li>
<li class="toctree-l1"><a class="reference internal" href="caching.html">Opportunistic Caching</a></li>
<li class="toctree-l1"><a class="reference internal" href="bytes.html">Internal Data Ingestion</a></li>
<li class="toctree-l1"><a class="reference internal" href="remote-data-services.html">Remote Data Services</a></li>
<li class="toctree-l1"><a class="reference internal" href="custom-collections.html">Custom Collections</a></li>
<li class="toctree-l1"><a class="reference internal" href="cite.html">Citations</a></li>
<li class="toctree-l1"><a class="reference internal" href="funding.html">Funding</a></li>
</ul>

            
          
        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="index.html">dask</a>
        
      </nav>


      
      <div class="wy-nav-content">
        <div class="rst-content">
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="index.html">Docs</a> &raquo;</li>
        
          <li><a href="bag.html">Bag</a> &raquo;</li>
        
      <li>Overview</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
            <a href="_sources/bag-overview.rst.txt" rel="nofollow"> View page source</a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="overview">
<h1>Overview<a class="headerlink" href="#overview" title="Permalink to this headline"></a></h1>
<p>Dask.Bag implements a operations like <code class="docutils literal"><span class="pre">map</span></code>, <code class="docutils literal"><span class="pre">filter</span></code>, <code class="docutils literal"><span class="pre">fold</span></code>, and
<code class="docutils literal"><span class="pre">groupby</span></code> on collections of Python objects.  It does this in parallel and in
small memory using Python iterators.  It is similar to a parallel version of
<a class="reference external" href="https://toolz.readthedocs.io/en/latest/">PyToolz</a> or a Pythonic version of the <a class="reference external" href="http://spark.apache.org/docs/latest/api/python/pyspark.html">PySpark RDD</a>.</p>
<div class="section" id="design">
<h2>Design<a class="headerlink" href="#design" title="Permalink to this headline"></a></h2>
<p>Dask bags coordinate many Python lists or Iterators, each of which forms a
partition of a larger collection.</p>
</div>
<div class="section" id="common-uses">
<h2>Common Uses<a class="headerlink" href="#common-uses" title="Permalink to this headline"></a></h2>
<p>Dask bags are often used to parallelize simple computations on unstructured or
semi-structured data like text data, log files, JSON records, or user defined
Python objects.</p>
</div>
<div class="section" id="execution">
<h2>Execution<a class="headerlink" href="#execution" title="Permalink to this headline"></a></h2>
<p>Execution on bags provide two benefits:</p>
<ol class="arabic simple">
<li>Parallel: data is split up, allowing multiple cores or machines to execute
in parallel.</li>
<li>Iterating: data processes lazily, allowing smooth execution of
larger-than-memory data, even on a single machine within a single partition</li>
</ol>
<div class="section" id="default-scheduler">
<h3>Default scheduler<a class="headerlink" href="#default-scheduler" title="Permalink to this headline"></a></h3>
<p>By default <code class="docutils literal"><span class="pre">dask.bag</span></code> uses <code class="docutils literal"><span class="pre">dask.multiprocessing</span></code> for computation.  As a
benefit Dask bypasses the <a class="reference external" href="https://docs.python.org/3/glossary.html#term-gil">GIL</a> and uses multiple cores on Pure Python objects.
As a drawback Dask.bag doesn’t perform well on computations that include a
great deal of inter-worker communication.  For common operations this is rarely
an issue as most Dask.bag workflows are embarrassingly parallel or result in
reductions with little data moving between workers.</p>
</div>
<div class="section" id="shuffle">
<h3>Shuffle<a class="headerlink" href="#shuffle" title="Permalink to this headline"></a></h3>
<p>Some operations, like <code class="docutils literal"><span class="pre">groupby</span></code>, require substantial inter-worker
communication. On a single machine, dask uses <a class="reference external" href="https://github.com/mrocklin/partd">partd</a> to perform efficient,
parallel, spill-to-disk shuffles. When working in a cluster, dask uses a task
based shuffle.</p>
<p>These shuffle operations are expensive and better handled by projects like
<code class="docutils literal"><span class="pre">dask.dataframe</span></code>. It is best to use <code class="docutils literal"><span class="pre">dask.bag</span></code> to clean and process data,
then transform it into an array or dataframe before embarking on the more
complex operations that require shuffle steps.</p>
</div>
</div>
<div class="section" id="known-limitations">
<h2>Known Limitations<a class="headerlink" href="#known-limitations" title="Permalink to this headline"></a></h2>
<p>Bags provide very general computation (any Python function.)  This generality
comes at cost.  Bags have the following known limitations:</p>
<ol class="arabic simple">
<li>By default they rely on the multiprocessing scheduler, which has its own
set of known limitations (see <a class="reference internal" href="shared.html"><span class="doc">Shared Memory</span></a>)</li>
<li>Bags are immutable and so you can not change individual elements</li>
<li>Bag operations tend to be slower than array/dataframe computations in the
same way that standard Python containers tend to be slower than NumPy
arrays and Pandas dataframes.</li>
<li><code class="docutils literal"><span class="pre">Bag.groupby</span></code> is slow.  You should try to use <code class="docutils literal"><span class="pre">Bag.foldby</span></code> if possible.
Using <code class="docutils literal"><span class="pre">Bag.foldby</span></code> requires more thought.</li>
</ol>
</div>
<div class="section" id="name">
<h2>Name<a class="headerlink" href="#name" title="Permalink to this headline"></a></h2>
<p><em>Bag</em> is the mathematical name for an unordered collection allowing repeats. It
is a friendly synonym to <a class="reference external" href="http://en.wikipedia.org/wiki/Bag_(mathematics)">multiset</a>. A bag or a multiset is a generalization of
the concept of a set that, unlike a set, allows multiple instances of the
multiset’s elements.</p>
<ul class="simple">
<li><code class="docutils literal"><span class="pre">list</span></code>: <em>ordered</em> collection <em>with repeats</em>, <code class="docutils literal"><span class="pre">[1,</span> <span class="pre">2,</span> <span class="pre">3,</span> <span class="pre">2]</span></code></li>
<li><code class="docutils literal"><span class="pre">set</span></code>: <em>unordered</em> collection <em>without repeats</em>,  <code class="docutils literal"><span class="pre">{1,</span> <span class="pre">2,</span> <span class="pre">3}</span></code></li>
<li><code class="docutils literal"><span class="pre">bag</span></code>: <em>unordered</em> collection <em>with repeats</em>, <code class="docutils literal"><span class="pre">{1,</span> <span class="pre">2,</span> <span class="pre">2,</span> <span class="pre">3}</span></code></li>
</ul>
<p>So a bag is like a list, but it doesn’t guarantee an ordering among elements.
There can be repeated elements but you can’t ask for the ith element.</p>
</div>
</div>


           </div>
           <div class="articleComments">
            
           </div>
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="bag-creation.html" class="btn btn-neutral float-right" title="Create Dask Bags" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="bag.html" class="btn btn-neutral" title="Bag" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2017, Anaconda.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  


  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'./',
            VERSION:'0.16.0',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true,
            SOURCELINK_SUFFIX: '.txt'
        };
    </script>
      <script type="text/javascript" src="_static/jquery.js"></script>
      <script type="text/javascript" src="_static/underscore.js"></script>
      <script type="text/javascript" src="_static/doctools.js"></script>
      <script type="text/javascript" src="file:///usr/share/javascript/mathjax/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>

  

  
  
    <script type="text/javascript" src="_static/js/theme.js"></script>
  

  
  
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.StickyNav.enable();
      });
  </script>
   

</body>
</html>

APT Browse - Built by Thomas Orozco.