/usr/include/irstlm/lmmacro.h is in libirstlm-dev 6.00.05-2.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 | // $Id: lmmacro.h 3461 2010-08-27 10:17:34Z bertoldi $
/******************************************************************************
IrstLM: IRST Language Model Toolkit
Copyright (C) 2006 Marcello Federico, ITC-irst Trento, Italy
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
******************************************************************************/
#ifndef MF_LMMACRO_H
#define MF_LMMACRO_H
#ifndef WIN32
#include <sys/types.h>
#include <sys/mman.h>
#endif
#include "util.h"
#include "ngramcache.h"
#include "dictionary.h"
#include "n_gram.h"
#include "lmtable.h"
#define MAX_TOKEN_N_MAP 5
namespace irstlm {
class lmmacro: public lmtable
{
dictionary *dict;
int maxlev; //max level of table
int selectedField;
bool collapseFlag; //flag for the presence of collapse
bool mapFlag; //flag for the presence of map
int microMacroMapN;
int *microMacroMap;
bool *collapsableMap;
bool *collapsatorMap;
#ifdef DLEXICALLM
int selectedFieldForLexicon;
int *lexicaltoken2classMap;
int lexicaltoken2classMapN;
#endif
void loadmap(const std::string mapfilename);
void unloadmap();
bool transform(ngram &in, ngram &out);
void field_selection(ngram &in, ngram &out);
bool collapse(ngram &in, ngram &out);
void mapping(ngram &in, ngram &out);
public:
lmmacro(float nlf=0.0, float dlfi=0.0);
~lmmacro();
virtual void load(const std::string &filename,int mmap=0);
virtual double lprob(ngram ng, double* bow, int* bol, ngram_state_t* maxsuffidx, char** maxsuffptr, unsigned int* statesize, bool* extendible, double* lastbow);
virtual double clprob(ngram ng, double* bow, int* bol, ngram_state_t* maxsuffidx, char** maxsuffptr, unsigned int* statesize, bool* extendible, double* lastbow);
virtual const char *maxsuffptr(ngram ong, unsigned int* size=NULL);
virtual ngram_state_t maxsuffidx(ngram ong, unsigned int* size=NULL);
void map(ngram *in, ngram *out);
void One2OneMapping(ngram *in, ngram *out);
void Micro2MacroMapping(ngram *in, ngram *out);
#ifdef DLEXICALLM
void Micro2MacroMapping(ngram *in, ngram *out, char **lemma);
void loadLexicalClasses(const char *fn);
void cutLex(ngram *in, ngram *out);
#endif
inline bool is_OOV(int code) {
ngram word_ng(getDict());
ngram field_ng(getDict());
word_ng.pushc(code);
if (selectedField >= 0)
field_selection(word_ng, field_ng);
else
field_ng = word_ng;
int field_code=*field_ng.wordp(1);
VERBOSE(2,"inline virtual bool lmmacro::is_OOV(int code) word_ng:" << word_ng << " field_ng:" << field_ng << std::endl);
//the selected field(s) of a token is considered OOV
//either if unknown by the microMacroMap
//or if its mapped macroW is OOV
if (field_code >= microMacroMapN) return true;
VERBOSE(2,"inline virtual bool lmmacro::is_OOV(int code)*field_code:" << field_code << " microMacroMap[field_code]:" << microMacroMap[field_code] << " lmtable::dict->oovcode():" << lmtable::dict->oovcode() << std::endl);
return (microMacroMap[field_code] == lmtable::dict->oovcode());
};
inline dictionary* getDict() const {
return dict;
}
inline int maxlevel() const {
return maxlev;
};
inline virtual void dictionary_incflag(const bool flag) {
dict->incflag(flag);
};
inline virtual bool filter(const string sfilter, lmContainer* sublmt, const string skeepunigrams) {
UNUSED(sfilter);
UNUSED(sublmt);
UNUSED(skeepunigrams);
return false;
}
};
}//namespace irstlm
#endif
|