/usr/include/mrmpi/keymultivalue.h is in libmrmpi-dev 1.0~20140404-2.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 | /* ----------------------------------------------------------------------
MR-MPI = MapReduce-MPI library
http://www.cs.sandia.gov/~sjplimp/mapreduce.html
Steve Plimpton, sjplimp@sandia.gov, Sandia National Laboratories
Copyright (2009) Sandia Corporation. Under the terms of Contract
DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
certain rights in this software. This software is distributed under
the modified Berkeley Software Distribution (BSD) License.
See the README file in the top-level MapReduce directory.
------------------------------------------------------------------------- */
#ifndef KEY_MULTIVALUE_H
#define KEY_MULTIVALUE_H
#include "mpi.h"
#include "stdio.h"
#include "stdint.h"
namespace MAPREDUCE_NS {
class KeyMultiValue {
public:
uint64_t nkmv; // # of KMV pairs in entire KMV on this proc
uint64_t ksize; // exact size of all key data
uint64_t vsize; // exact size of all multivalue data
uint64_t esize; // total exact size of entire KMV
uint64_t fsize; // size of KMV file
char *page; // in-memory page
int memtag; // memory page ID
int npage; // # of pages in entire KMV
KeyMultiValue(class MapReduce *, int, int,
class Memory *, class Error *, MPI_Comm);
~KeyMultiValue();
void allocate();
void deallocate(int);
void copy(KeyMultiValue *);
void complete();
int request_info(char **);
int request_page(int, int, uint64_t &, uint64_t &, uint64_t &);
uint64_t multivalue_blocks(int, int &);
void overwrite_page(int);
void close_file();
void clone(class KeyValue *);
void collapse(char *, int, class KeyValue *);
void convert(class KeyValue *);
void print(FILE *, int, int, int);
private:
MapReduce *mr;
MPI_Comm comm;
class Memory *memory;
class Error *error;
int me;
uint64_t pagesize; // size of page
int kalign,valign; // alignment for keys & multivalues
int talign; // alignment of entire KMV pair
int ualign; // alignment of Unique
int kalignm1,valignm1; // alignments-1 for masking
int talignm1,ualignm1;
int twolenbytes; // size of key & value lengths
int threelenbytes; // size of nvalue & key & value lengths
// in-memory page
int nkey; // # of KMV pairs in page
uint64_t nvalue; // # of values in all KMV mvalues in page
uint64_t keysize; // exact size of key data in page
uint64_t valuesize; // exact size of multivalue data in page
uint64_t alignsize; // current size of page with alignment
// virtual pages
struct Page {
uint64_t keysize; // exact size of keys
uint64_t valuesize; // exact size of multivalues
uint64_t exactsize; // exact size of all data in page
uint64_t alignsize; // aligned size of all data in page
uint64_t filesize; // rounded-up alignsize for file I/O
uint64_t fileoffset; // summed filesize of all previous pages
uint64_t nvalue_total; // total # of values for multi-page KMV header
int nkey; // # of KMV pairs
int nblock; // # of value blocks for multi-page KMV header
};
Page *pages; // list of pages
int maxpage; // max # of pages currently allocated
// unique keys
int nunique; // current # of unique keys
int ukeyoffset; // offset from start of Unique to where key starts
struct Unique {
uint64_t nvalue; // # of values associated with this key
uint64_t mvbytes; // total size of values associated with this key
int *soffset; // ptr to start of value sizes in KMV page
char *voffset; // ptr to start of values in KMV page
Unique *next; // ptr to next key in this hash bucket
int keybytes; // size of this key
int set; // which KMV set this key will be part of
};
// hash of unique keys
Unique **buckets; // ptr to 1st key in each hash bucket
int hashmask; // bit mask for mapping hashed key into hash buckets
// nbuckets = hashmask + 1
uint64_t bucketbytes; // byte size of hash buckets
char *memunique; // ptr to where memory for hash+Uniques starts
char *ustart; // ptr to where memory for Uniques starts
char *ustop; // ptr to where memory for Uniques stops
// file info
int fileflag; // 1 if file exists, 0 if not
char *filename; // filename to store KMV if needed
FILE *fp; // file ptr
// partitions of KV data per unique list
struct Partition {
class KeyValue *kv; // primary KV storing pairs for this partition
class Spool *sp; // secondary Spool of pairs if re-partitioned
class Spool *sp2; // tertiary Spool of pairs if re-partitioned
int sortbit; // bit from hi-end that partitioning was done on
};
Partition *partitions;
int npartition,maxpartition;
// sets of unique keys per KMV page
struct Set {
class KeyValue *kv; // KV pairs for set can be in KV and/or Spool(s)
class Spool *sp;
class Spool *sp2;
Unique *first; // ptr to first Unique in set
int nunique; // # of Uniques in set
int extended; // 1 if set contains one Unique -> multi-page KMV
};
Set *sets;
int nset,maxset;
// memory management for Spool pages
char *readpage; // convert() does all reading from this page
int minspool; // minimum allowed size for a spool page
int npages_mr; // # of MR pages I have allocated
int *tag_mr; // page IDs for MR pages
char **page_mr; // ptrs to MR pages
uint64_t sizespool; // size of spool page
int spoolperpage; // # of spool pages per MR page
int nquery; // # of requested spool pages on this iteration
// private methods
void add(char *, int, char *, int);
void collapse_one(char *, int, class KeyValue *, uint64_t);
void collapse_many(char *, int, class KeyValue *);
void kv2unique(int);
int unique2kmv_all();
void unique2kmv_extended(int);
void unique2kmv_set(int);
void partition2sets(int);
void kv2kmv(int);
void kv2kmv_extended(int);
class Spool *augment_partition(int);
class Spool *create_partition(int);
char *chunk_allocate();
Unique *find(int, char *, int, Unique *&);
int hash(char *, int);
void init_page();
void create_page();
void write_page();
void read_page(int, int);
uint64_t roundup(uint64_t, int);
void spool_memory(class KeyValue *);
void spool_request(int, int);
char *spool_malloc(int, uint64_t &);
void spool_free();
};
}
#endif
|