\doxysection{ngram\+\_\+model\+\_\+set.\+c}
\label{ngram__model__set_8c_source}\index{src/libsphinxbase/lm/ngram\_model\_set.c@{src/libsphinxbase/lm/ngram\_model\_set.c}}
\textbf{ Go to the documentation of this file.}
\begin{DoxyCode}{0}
\DoxyCodeLine{00001\ \textcolor{comment}{/*\ -\/*-\/\ c-\/basic-\/offset:\ 4;\ indent-\/tabs-\/mode:\ nil\ -\/*-\/\ */}}
\DoxyCodeLine{00002\ \textcolor{comment}{/*\ ====================================================================}}
\DoxyCodeLine{00003\ \textcolor{comment}{\ *\ Copyright\ (c)\ 2008\ Carnegie\ Mellon\ University.\ \ All\ rights}}
\DoxyCodeLine{00004\ \textcolor{comment}{\ *\ reserved.}}
\DoxyCodeLine{00005\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00006\ \textcolor{comment}{\ *\ Redistribution\ and\ use\ in\ source\ and\ binary\ forms,\ with\ or\ without}}
\DoxyCodeLine{00007\ \textcolor{comment}{\ *\ modification,\ are\ permitted\ provided\ that\ the\ following\ conditions}}
\DoxyCodeLine{00008\ \textcolor{comment}{\ *\ are\ met:}}
\DoxyCodeLine{00009\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00010\ \textcolor{comment}{\ *\ 1.\ Redistributions\ of\ source\ code\ must\ retain\ the\ above\ copyright}}
\DoxyCodeLine{00011\ \textcolor{comment}{\ *\ \ \ \ notice,\ this\ list\ of\ conditions\ and\ the\ following\ disclaimer.\ }}
\DoxyCodeLine{00012\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00013\ \textcolor{comment}{\ *\ 2.\ Redistributions\ in\ binary\ form\ must\ reproduce\ the\ above\ copyright}}
\DoxyCodeLine{00014\ \textcolor{comment}{\ *\ \ \ \ notice,\ this\ list\ of\ conditions\ and\ the\ following\ disclaimer\ in}}
\DoxyCodeLine{00015\ \textcolor{comment}{\ *\ \ \ \ the\ documentation\ and/or\ other\ materials\ provided\ with\ the}}
\DoxyCodeLine{00016\ \textcolor{comment}{\ *\ \ \ \ distribution.}}
\DoxyCodeLine{00017\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00018\ \textcolor{comment}{\ *\ This\ work\ was\ supported\ in\ part\ by\ funding\ from\ the\ Defense\ Advanced\ }}
\DoxyCodeLine{00019\ \textcolor{comment}{\ *\ Research\ Projects\ Agency\ and\ the\ National\ Science\ Foundation\ of\ the\ }}
\DoxyCodeLine{00020\ \textcolor{comment}{\ *\ United\ States\ of\ America,\ and\ the\ CMU\ Sphinx\ Speech\ Consortium.}}
\DoxyCodeLine{00021\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00022\ \textcolor{comment}{\ *\ THIS\ SOFTWARE\ IS\ PROVIDED\ BY\ CARNEGIE\ MELLON\ UNIVERSITY\ \`{}\`{}AS\ IS''\ AND\ }}
\DoxyCodeLine{00023\ \textcolor{comment}{\ *\ ANY\ EXPRESSED\ OR\ IMPLIED\ WARRANTIES,\ INCLUDING,\ BUT\ NOT\ LIMITED\ TO,\ }}
\DoxyCodeLine{00024\ \textcolor{comment}{\ *\ THE\ IMPLIED\ WARRANTIES\ OF\ MERCHANTABILITY\ AND\ FITNESS\ FOR\ A\ PARTICULAR}}
\DoxyCodeLine{00025\ \textcolor{comment}{\ *\ PURPOSE\ ARE\ DISCLAIMED.\ \ IN\ NO\ EVENT\ SHALL\ CARNEGIE\ MELLON\ UNIVERSITY}}
\DoxyCodeLine{00026\ \textcolor{comment}{\ *\ NOR\ ITS\ EMPLOYEES\ BE\ LIABLE\ FOR\ ANY\ DIRECT,\ INDIRECT,\ INCIDENTAL,}}
\DoxyCodeLine{00027\ \textcolor{comment}{\ *\ SPECIAL,\ EXEMPLARY,\ OR\ CONSEQUENTIAL\ DAMAGES\ (INCLUDING,\ BUT\ NOT\ }}
\DoxyCodeLine{00028\ \textcolor{comment}{\ *\ LIMITED\ TO,\ PROCUREMENT\ OF\ SUBSTITUTE\ GOODS\ OR\ SERVICES;\ LOSS\ OF\ USE,\ }}
\DoxyCodeLine{00029\ \textcolor{comment}{\ *\ DATA,\ OR\ PROFITS;\ OR\ BUSINESS\ INTERRUPTION)\ HOWEVER\ CAUSED\ AND\ ON\ ANY\ }}
\DoxyCodeLine{00030\ \textcolor{comment}{\ *\ THEORY\ OF\ LIABILITY,\ WHETHER\ IN\ CONTRACT,\ STRICT\ LIABILITY,\ OR\ TORT\ }}
\DoxyCodeLine{00031\ \textcolor{comment}{\ *\ (INCLUDING\ NEGLIGENCE\ OR\ OTHERWISE)\ ARISING\ IN\ ANY\ WAY\ OUT\ OF\ THE\ USE\ }}
\DoxyCodeLine{00032\ \textcolor{comment}{\ *\ OF\ THIS\ SOFTWARE,\ EVEN\ IF\ ADVISED\ OF\ THE\ POSSIBILITY\ OF\ SUCH\ DAMAGE.}}
\DoxyCodeLine{00033\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00034\ \textcolor{comment}{\ *\ ====================================================================}}
\DoxyCodeLine{00035\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00036\ \textcolor{comment}{\ */}}
\DoxyCodeLine{00042\ \textcolor{preprocessor}{\#include\ <string.h>}}
\DoxyCodeLine{00043\ \textcolor{preprocessor}{\#include\ <stdlib.h>}}
\DoxyCodeLine{00044\ }
\DoxyCodeLine{00045\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/err.h"{}}}
\DoxyCodeLine{00046\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/ckd\_alloc.h"{}}}
\DoxyCodeLine{00047\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/strfuncs.h"{}}}
\DoxyCodeLine{00048\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/filename.h"{}}}
\DoxyCodeLine{00049\ }
\DoxyCodeLine{00050\ \textcolor{preprocessor}{\#include\ "{}ngram\_model\_set.h"{}}}
\DoxyCodeLine{00051\ }
\DoxyCodeLine{00052\ \textcolor{keyword}{static}\ ngram\_funcs\_t\ ngram\_model\_set\_funcs;}
\DoxyCodeLine{00053\ }
\DoxyCodeLine{00054\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{int}}
\DoxyCodeLine{00055\ my\_compare(\textcolor{keyword}{const}\ \textcolor{keywordtype}{void}\ *a,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{void}\ *b)}
\DoxyCodeLine{00056\ \{}
\DoxyCodeLine{00057\ \ \ \ \ \textcolor{comment}{/*\ Make\ sure\ <UNK>\ floats\ to\ the\ beginning.\ */}}
\DoxyCodeLine{00058\ \ \ \ \ \textcolor{keywordflow}{if}\ (strcmp(*(\textcolor{keywordtype}{char}\ *\ \textcolor{keyword}{const}\ *)a,\ \textcolor{stringliteral}{"{}<UNK>"{}})\ ==\ 0)}
\DoxyCodeLine{00059\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00060\ \ \ \ \ \textcolor{keywordflow}{else}\ \textcolor{keywordflow}{if}\ (strcmp(*(\textcolor{keywordtype}{char}\ *\ \textcolor{keyword}{const}\ *)b,\ \textcolor{stringliteral}{"{}<UNK>"{}})\ ==\ 0)}
\DoxyCodeLine{00061\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ 1;}
\DoxyCodeLine{00062\ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00063\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ strcmp(*(\textcolor{keywordtype}{char}\ *\ \textcolor{keyword}{const}\ *)a,\ *(\textcolor{keywordtype}{char}\ *\ \textcolor{keyword}{const}\ *)b);}
\DoxyCodeLine{00064\ \}}
\DoxyCodeLine{00065\ }
\DoxyCodeLine{00066\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00067\ build\_widmap(ngram\_model\_t\ *base,\ logmath\_t\ *lmath,\ int32\ n)}
\DoxyCodeLine{00068\ \{}
\DoxyCodeLine{00069\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00070\ \ \ \ \ ngram\_model\_t\ **models\ =\ set-\/>lms;}
\DoxyCodeLine{00071\ \ \ \ \ hash\_table\_t\ *vocab;}
\DoxyCodeLine{00072\ \ \ \ \ glist\_t\ hlist;}
\DoxyCodeLine{00073\ \ \ \ \ gnode\_t\ *gn;}
\DoxyCodeLine{00074\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00075\ }
\DoxyCodeLine{00076\ \ \ \ \ \textcolor{comment}{/*\ Construct\ a\ merged\ vocabulary\ and\ a\ set\ of\ word-\/ID\ mappings.\ */}}
\DoxyCodeLine{00077\ \ \ \ \ vocab\ =\ hash\_table\_new(models[0]-\/>n\_words,\ FALSE);}
\DoxyCodeLine{00078\ \ \ \ \ \textcolor{comment}{/*\ Create\ the\ set\ of\ merged\ words.\ */}}
\DoxyCodeLine{00079\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ set-\/>n\_models;\ ++i)\ \{}
\DoxyCodeLine{00080\ \ \ \ \ \ \ \ \ int32\ j;}
\DoxyCodeLine{00081\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (j\ =\ 0;\ j\ <\ models[i]-\/>n\_words;\ ++j)\ \{}
\DoxyCodeLine{00082\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Ignore\ collisions.\ */}}
\DoxyCodeLine{00083\ \ \ \ \ \ \ \ \ \ \ \ \ (void)hash\_table\_enter\_int32(vocab,\ models[i]-\/>word\_str[j],\ j);}
\DoxyCodeLine{00084\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00085\ \ \ \ \ \}}
\DoxyCodeLine{00086\ \ \ \ \ \textcolor{comment}{/*\ Create\ the\ array\ of\ words,\ then\ sort\ it.\ */}}
\DoxyCodeLine{00087\ \ \ \ \ \textcolor{keywordflow}{if}\ (hash\_table\_lookup(vocab,\ \textcolor{stringliteral}{"{}<UNK>"{}},\ NULL)\ !=\ 0)}
\DoxyCodeLine{00088\ \ \ \ \ \ \ \ \ (\textcolor{keywordtype}{void})hash\_table\_enter\_int32(vocab,\ \textcolor{stringliteral}{"{}<UNK>"{}},\ 0);}
\DoxyCodeLine{00089\ \ \ \ \ \textcolor{comment}{/*\ Now\ we\ know\ the\ number\ of\ unigrams,\ initialize\ the\ base\ model.\ */}}
\DoxyCodeLine{00090\ \ \ \ \ ngram\_model\_init(base,\ \&ngram\_model\_set\_funcs,\ lmath,\ n,\ hash\_table\_inuse(vocab));}
\DoxyCodeLine{00091\ \ \ \ \ base-\/>writable\ =\ FALSE;\ \textcolor{comment}{/*\ We\ will\ reuse\ the\ pointers\ from\ the\ submodels.\ */}}
\DoxyCodeLine{00092\ \ \ \ \ i\ =\ 0;}
\DoxyCodeLine{00093\ \ \ \ \ hlist\ =\ hash\_table\_tolist(vocab,\ NULL);}
\DoxyCodeLine{00094\ \ \ \ \ \textcolor{keywordflow}{for}\ (gn\ =\ hlist;\ gn;\ gn\ =\ gnode\_next(gn))\ \{}
\DoxyCodeLine{00095\ \ \ \ \ \ \ \ \ hash\_entry\_t\ *ent\ =\ gnode\_ptr(gn);}
\DoxyCodeLine{00096\ \ \ \ \ \ \ \ \ base-\/>word\_str[i++]\ =\ (\textcolor{keywordtype}{char}\ *)ent-\/>key;}
\DoxyCodeLine{00097\ \ \ \ \ \}}
\DoxyCodeLine{00098\ \ \ \ \ glist\_free(hlist);}
\DoxyCodeLine{00099\ \ \ \ \ qsort(base-\/>word\_str,\ base-\/>n\_words,\ \textcolor{keyword}{sizeof}(*base-\/>word\_str),\ my\_compare);}
\DoxyCodeLine{00100\ }
\DoxyCodeLine{00101\ \ \ \ \ \textcolor{comment}{/*\ Now\ create\ the\ word\ ID\ mappings.\ */}}
\DoxyCodeLine{00102\ \ \ \ \ \textcolor{keywordflow}{if}\ (set-\/>widmap)}
\DoxyCodeLine{00103\ \ \ \ \ \ \ \ \ ckd\_free\_2d((\textcolor{keywordtype}{void}\ **)set-\/>widmap);}
\DoxyCodeLine{00104\ \ \ \ \ set-\/>widmap\ =\ (int32\ **)\ ckd\_calloc\_2d(base-\/>n\_words,\ set-\/>n\_models,}
\DoxyCodeLine{00105\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{sizeof}(**set-\/>widmap));}
\DoxyCodeLine{00106\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ base-\/>n\_words;\ ++i)\ \{}
\DoxyCodeLine{00107\ \ \ \ \ \ \ \ \ int32\ j;}
\DoxyCodeLine{00108\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Also\ create\ the\ master\ wid\ mapping.\ */}}
\DoxyCodeLine{00109\ \ \ \ \ \ \ \ \ (void)hash\_table\_enter\_int32(base-\/>wid,\ base-\/>word\_str[i],\ i);}
\DoxyCodeLine{00110\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ printf("{}\%s:\ \%d\ =>\ "{},\ base-\/>word\_str[i],\ i);\ */}}
\DoxyCodeLine{00111\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (j\ =\ 0;\ j\ <\ set-\/>n\_models;\ ++j)\ \{}
\DoxyCodeLine{00112\ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>widmap[i][j]\ =\ ngram\_wid(models[j],\ base-\/>word\_str[i]);}
\DoxyCodeLine{00113\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ printf("{}\%d\ "{},\ set-\/>widmap[i][j]);\ */}}
\DoxyCodeLine{00114\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00115\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ printf("{}\(\backslash\)n"{});\ */}}
\DoxyCodeLine{00116\ \ \ \ \ \}}
\DoxyCodeLine{00117\ \ \ \ \ hash\_table\_free(vocab);}
\DoxyCodeLine{00118\ \}}
\DoxyCodeLine{00119\ }
\DoxyCodeLine{00120\ ngram\_model\_t\ *}
\DoxyCodeLine{00121\ ngram\_model\_set\_init(cmd\_ln\_t\ *config,}
\DoxyCodeLine{00122\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_model\_t\ **models,}
\DoxyCodeLine{00123\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{char}\ **names,}
\DoxyCodeLine{00124\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ float32\ *weights,}
\DoxyCodeLine{00125\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ n\_models)}
\DoxyCodeLine{00126\ \{}
\DoxyCodeLine{00127\ \ \ \ \ ngram\_model\_set\_t\ *model;}
\DoxyCodeLine{00128\ \ \ \ \ ngram\_model\_t\ *base;}
\DoxyCodeLine{00129\ \ \ \ \ logmath\_t\ *lmath;}
\DoxyCodeLine{00130\ \ \ \ \ int32\ i,\ n;}
\DoxyCodeLine{00131\ }
\DoxyCodeLine{00132\ \ \ \ \ \textcolor{keywordflow}{if}\ (n\_models\ ==\ 0)\ \textcolor{comment}{/*\ WTF\ */}}
\DoxyCodeLine{00133\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00134\ }
\DoxyCodeLine{00135\ \ \ \ \ \textcolor{comment}{/*\ Do\ consistency\ checking\ on\ the\ models.\ \ They\ must\ all\ use\ the}}
\DoxyCodeLine{00136\ \textcolor{comment}{\ \ \ \ \ *\ same\ logbase\ and\ shift.\ */}}
\DoxyCodeLine{00137\ \ \ \ \ lmath\ =\ models[0]-\/>lmath;}
\DoxyCodeLine{00138\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 1;\ i\ <\ n\_models;\ ++i)\ \{}
\DoxyCodeLine{00139\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (logmath\_get\_base(models[i]-\/>lmath)\ !=\ logmath\_get\_base(lmath)}
\DoxyCodeLine{00140\ \ \ \ \ \ \ \ \ \ \ \ \ ||\ logmath\_get\_shift(models[i]-\/>lmath)\ !=\ logmath\_get\_shift(lmath))\ \{}
\DoxyCodeLine{00141\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Log-\/math\ parameters\ don't\ match,\ will\ not\ create\ LM\ set\(\backslash\)n"{}});}
\DoxyCodeLine{00142\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00143\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00144\ \ \ \ \ \}}
\DoxyCodeLine{00145\ }
\DoxyCodeLine{00146\ \ \ \ \ \textcolor{comment}{/*\ Allocate\ the\ combined\ model,\ initialize\ it.\ */}}
\DoxyCodeLine{00147\ \ \ \ \ model\ =\ ckd\_calloc(1,\ \textcolor{keyword}{sizeof}(*model));}
\DoxyCodeLine{00148\ \ \ \ \ base\ =\ \&model-\/>base;}
\DoxyCodeLine{00149\ \ \ \ \ model-\/>n\_models\ =\ n\_models;}
\DoxyCodeLine{00150\ \ \ \ \ model-\/>lms\ =\ ckd\_calloc(n\_models,\ \textcolor{keyword}{sizeof}(*model-\/>lms));}
\DoxyCodeLine{00151\ \ \ \ \ model-\/>names\ =\ ckd\_calloc(n\_models,\ \textcolor{keyword}{sizeof}(*model-\/>names));}
\DoxyCodeLine{00152\ \ \ \ \ \textcolor{comment}{/*\ Initialize\ weights\ to\ a\ uniform\ distribution\ */}}
\DoxyCodeLine{00153\ \ \ \ \ model-\/>lweights\ =\ ckd\_calloc(n\_models,\ \textcolor{keyword}{sizeof}(*model-\/>lweights));}
\DoxyCodeLine{00154\ \ \ \ \ \{}
\DoxyCodeLine{00155\ \ \ \ \ \ \ \ \ int32\ uniform\ =\ logmath\_log(lmath,\ 1.0/n\_models);}
\DoxyCodeLine{00156\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ n\_models;\ ++i)}
\DoxyCodeLine{00157\ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>lweights[i]\ =\ uniform;}
\DoxyCodeLine{00158\ \ \ \ \ \}}
\DoxyCodeLine{00159\ \ \ \ \ \textcolor{comment}{/*\ Default\ to\ interpolate\ if\ weights\ were\ given.\ */}}
\DoxyCodeLine{00160\ \ \ \ \ \textcolor{keywordflow}{if}\ (weights)}
\DoxyCodeLine{00161\ \ \ \ \ \ \ \ \ model-\/>cur\ =\ -\/1;}
\DoxyCodeLine{00162\ }
\DoxyCodeLine{00163\ \ \ \ \ n\ =\ 0;}
\DoxyCodeLine{00164\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ n\_models;\ ++i)\ \{}
\DoxyCodeLine{00165\ \ \ \ \ \ \ \ \ model-\/>lms[i]\ =\ models[i];}
\DoxyCodeLine{00166\ \ \ \ \ \ \ \ \ model-\/>names[i]\ =\ ckd\_salloc(names[i]);}
\DoxyCodeLine{00167\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (weights)}
\DoxyCodeLine{00168\ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>lweights[i]\ =\ logmath\_log(lmath,\ weights[i]);}
\DoxyCodeLine{00169\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ N\ is\ the\ maximum\ of\ all\ merged\ models.\ */}}
\DoxyCodeLine{00170\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (models[i]-\/>n\ >\ n)}
\DoxyCodeLine{00171\ \ \ \ \ \ \ \ \ \ \ \ \ n\ =\ models[i]-\/>n;}
\DoxyCodeLine{00172\ \ \ \ \ \}}
\DoxyCodeLine{00173\ \ \ \ \ \textcolor{comment}{/*\ Allocate\ the\ history\ mapping\ table.\ */}}
\DoxyCodeLine{00174\ \ \ \ \ model-\/>maphist\ =\ ckd\_calloc(n\ -\/\ 1,\ \textcolor{keyword}{sizeof}(*model-\/>maphist));}
\DoxyCodeLine{00175\ }
\DoxyCodeLine{00176\ \ \ \ \ \textcolor{comment}{/*\ Now\ build\ the\ word-\/ID\ mapping\ and\ merged\ vocabulary.\ */}}
\DoxyCodeLine{00177\ \ \ \ \ build\_widmap(base,\ lmath,\ n);}
\DoxyCodeLine{00178\ \ \ \ \ \textcolor{keywordflow}{return}\ base;}
\DoxyCodeLine{00179\ \}}
\DoxyCodeLine{00180\ }
\DoxyCodeLine{00181\ ngram\_model\_t\ *}
\DoxyCodeLine{00182\ ngram\_model\_set\_read(cmd\_ln\_t\ *config,}
\DoxyCodeLine{00183\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *lmctlfile,}
\DoxyCodeLine{00184\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ logmath\_t\ *lmath)}
\DoxyCodeLine{00185\ \{}
\DoxyCodeLine{00186\ \ \ \ \ FILE\ *ctlfp;}
\DoxyCodeLine{00187\ \ \ \ \ glist\_t\ lms\ =\ NULL;}
\DoxyCodeLine{00188\ \ \ \ \ glist\_t\ lmnames\ =\ NULL;}
\DoxyCodeLine{00189\ \ \ \ \ \_\_BIGSTACKVARIABLE\_\_\ \textcolor{keywordtype}{char}\ str[1024];}
\DoxyCodeLine{00190\ \ \ \ \ ngram\_model\_t\ *set\ =\ NULL;}
\DoxyCodeLine{00191\ \ \ \ \ hash\_table\_t\ *classes;}
\DoxyCodeLine{00192\ \ \ \ \ \textcolor{keywordtype}{char}\ *basedir,\ *c;}
\DoxyCodeLine{00193\ }
\DoxyCodeLine{00194\ \ \ \ \ \textcolor{comment}{/*\ Read\ all\ the\ class\ definition\ files\ to\ accumulate\ a\ mapping\ of}}
\DoxyCodeLine{00195\ \textcolor{comment}{\ \ \ \ \ *\ classnames\ to\ definitions.\ */}}
\DoxyCodeLine{00196\ \ \ \ \ classes\ =\ hash\_table\_new(0,\ FALSE);}
\DoxyCodeLine{00197\ \ \ \ \ \textcolor{keywordflow}{if}\ ((ctlfp\ =\ fopen(lmctlfile,\ \textcolor{stringliteral}{"{}r"{}}))\ ==\ NULL)\ \{}
\DoxyCodeLine{00198\ \ \ \ \ \ \ \ \ E\_ERROR\_SYSTEM(\textcolor{stringliteral}{"{}Failed\ to\ open\ \%s"{}},\ lmctlfile);}
\DoxyCodeLine{00199\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00200\ \ \ \ \ \}}
\DoxyCodeLine{00201\ }
\DoxyCodeLine{00202\ \ \ \ \ \textcolor{comment}{/*\ Try\ to\ find\ the\ base\ directory\ to\ append\ to\ relative\ paths\ in}}
\DoxyCodeLine{00203\ \textcolor{comment}{\ \ \ \ \ *\ the\ lmctl\ file.\ */}}
\DoxyCodeLine{00204\ \ \ \ \ \textcolor{keywordflow}{if}\ ((c\ =\ strrchr(lmctlfile,\ \textcolor{charliteral}{'/'}))\ ||\ (c\ =\ strrchr(lmctlfile,\ \textcolor{charliteral}{'\(\backslash\)\(\backslash\)'})))\ \{}
\DoxyCodeLine{00205\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Include\ the\ trailing\ slash.\ */}}
\DoxyCodeLine{00206\ \ \ \ \ \ \ \ \ basedir\ =\ ckd\_calloc(c\ -\/\ lmctlfile\ +\ 2,\ 1);}
\DoxyCodeLine{00207\ \ \ \ \ \ \ \ \ memcpy(basedir,\ lmctlfile,\ c\ -\/\ lmctlfile\ +\ 1);}
\DoxyCodeLine{00208\ \ \ \ \ \}}
\DoxyCodeLine{00209\ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00210\ \ \ \ \ \ \ \ \ basedir\ =\ NULL;}
\DoxyCodeLine{00211\ \ \ \ \ \}}
\DoxyCodeLine{00212\ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}Reading\ LM\ control\ file\ '\%s'\(\backslash\)n"{}},\ lmctlfile);}
\DoxyCodeLine{00213\ \ \ \ \ \textcolor{keywordflow}{if}\ (basedir)}
\DoxyCodeLine{00214\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}Will\ prepend\ '\%s'\ to\ unqualified\ paths\(\backslash\)n"{}},\ basedir);}
\DoxyCodeLine{00215\ }
\DoxyCodeLine{00216\ \ \ \ \ \textcolor{keywordflow}{if}\ (fscanf(ctlfp,\ \textcolor{stringliteral}{"{}\%1023s"{}},\ str)\ ==\ 1)\ \{}
\DoxyCodeLine{00217\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (strcmp(str,\ \textcolor{stringliteral}{"{}\{"{}})\ ==\ 0)\ \{}
\DoxyCodeLine{00218\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Load\ LMclass\ files\ */}}
\DoxyCodeLine{00219\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{while}\ ((fscanf(ctlfp,\ \textcolor{stringliteral}{"{}\%1023s"{}},\ str)\ ==\ 1)}
\DoxyCodeLine{00220\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \&\&\ (strcmp(str,\ \textcolor{stringliteral}{"{}\}"{}})\ !=\ 0))\ \{}
\DoxyCodeLine{00221\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{char}\ *deffile;}
\DoxyCodeLine{00222\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (basedir\ \&\&\ !path\_is\_absolute(str))}
\DoxyCodeLine{00223\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ deffile\ =\ string\_join(basedir,\ str,\ NULL);}
\DoxyCodeLine{00224\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00225\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ deffile\ =\ ckd\_salloc(str);}
\DoxyCodeLine{00226\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}Reading\ classdef\ from\ '\%s'\(\backslash\)n"{}},\ deffile);}
\DoxyCodeLine{00227\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (read\_classdef\_file(classes,\ deffile)\ <\ 0)\ \{}
\DoxyCodeLine{00228\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_free(deffile);}
\DoxyCodeLine{00229\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00230\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00231\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_free(deffile);}
\DoxyCodeLine{00232\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00233\ }
\DoxyCodeLine{00234\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (strcmp(str,\ \textcolor{stringliteral}{"{}\}"{}})\ !=\ 0)\ \{}
\DoxyCodeLine{00235\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Unexpected\ EOF\ in\ \%s\(\backslash\)n"{}},\ lmctlfile);}
\DoxyCodeLine{00236\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00237\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00238\ }
\DoxyCodeLine{00239\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ This\ might\ be\ the\ first\ LM\ name.\ */}}
\DoxyCodeLine{00240\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fscanf(ctlfp,\ \textcolor{stringliteral}{"{}\%1023s"{}},\ str)\ !=\ 1)}
\DoxyCodeLine{00241\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ str[0]\ =\ \textcolor{charliteral}{'\(\backslash\)0'};}
\DoxyCodeLine{00242\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00243\ \ \ \ \ \}}
\DoxyCodeLine{00244\ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00245\ \ \ \ \ \ \ \ \ str[0]\ =\ \textcolor{charliteral}{'\(\backslash\)0'};}
\DoxyCodeLine{00246\ }
\DoxyCodeLine{00247\ \ \ \ \ \textcolor{comment}{/*\ Read\ in\ one\ LM\ at\ a\ time\ and\ add\ classes\ to\ them\ as\ necessary.\ */}}
\DoxyCodeLine{00248\ \ \ \ \ \textcolor{keywordflow}{while}\ (str[0]\ !=\ \textcolor{charliteral}{'\(\backslash\)0'})\ \{}
\DoxyCodeLine{00249\ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{char}\ *lmfile;}
\DoxyCodeLine{00250\ \ \ \ \ \ \ \ \ ngram\_model\_t\ *lm;}
\DoxyCodeLine{00251\ }
\DoxyCodeLine{00252\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (basedir\ \&\&\ str[0]\ !=\ \textcolor{charliteral}{'/'}\ \&\&\ str[0]\ !=\ \textcolor{charliteral}{'\(\backslash\)\(\backslash\)'})}
\DoxyCodeLine{00253\ \ \ \ \ \ \ \ \ \ \ \ \ lmfile\ =\ string\_join(basedir,\ str,\ NULL);}
\DoxyCodeLine{00254\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00255\ \ \ \ \ \ \ \ \ \ \ \ \ lmfile\ =\ ckd\_salloc(str);}
\DoxyCodeLine{00256\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}Reading\ lm\ from\ '\%s'\(\backslash\)n"{}},\ lmfile);}
\DoxyCodeLine{00257\ \ \ \ \ \ \ \ \ lm\ =\ ngram\_model\_read(config,\ lmfile,\ NGRAM\_AUTO,\ lmath);}
\DoxyCodeLine{00258\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (lm\ ==\ NULL)\ \{}
\DoxyCodeLine{00259\ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_free(lmfile);}
\DoxyCodeLine{00260\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00261\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00262\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fscanf(ctlfp,\ \textcolor{stringliteral}{"{}\%1023s"{}},\ str)\ !=\ 1)\ \{}
\DoxyCodeLine{00263\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}LMname\ missing\ after\ LMFileName\ '\%s'\(\backslash\)n"{}},\ lmfile);}
\DoxyCodeLine{00264\ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_free(lmfile);}
\DoxyCodeLine{00265\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00266\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00267\ \ \ \ \ \ \ \ \ ckd\_free(lmfile);}
\DoxyCodeLine{00268\ \ \ \ \ \ \ \ \ lms\ =\ glist\_add\_ptr(lms,\ lm);}
\DoxyCodeLine{00269\ \ \ \ \ \ \ \ \ lmnames\ =\ glist\_add\_ptr(lmnames,\ ckd\_salloc(str));}
\DoxyCodeLine{00270\ }
\DoxyCodeLine{00271\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fscanf(ctlfp,\ \textcolor{stringliteral}{"{}\%1023s"{}},\ str)\ ==\ 1)\ \{}
\DoxyCodeLine{00272\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (strcmp(str,\ \textcolor{stringliteral}{"{}\{"{}})\ ==\ 0)\ \{}
\DoxyCodeLine{00273\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ LM\ uses\ classes;\ read\ their\ names\ */}}
\DoxyCodeLine{00274\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{while}\ ((fscanf(ctlfp,\ \textcolor{stringliteral}{"{}\%1023s"{}},\ str)\ ==\ 1)\ \&\&}
\DoxyCodeLine{00275\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (strcmp(str,\ \textcolor{stringliteral}{"{}\}"{}})\ !=\ 0))\ \{}
\DoxyCodeLine{00276\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{void}\ *val;}
\DoxyCodeLine{00277\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classdef\_t\ *classdef;}
\DoxyCodeLine{00278\ }
\DoxyCodeLine{00279\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (hash\_table\_lookup(classes,\ str,\ \&val)\ ==\ -\/1)\ \{}
\DoxyCodeLine{00280\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Unknown\ class\ \%s\ in\ control\ file\(\backslash\)n"{}},\ str);}
\DoxyCodeLine{00281\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00282\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00283\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classdef\ =\ val;}
\DoxyCodeLine{00284\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (ngram\_model\_add\_class(lm,\ str,\ 1.0,}
\DoxyCodeLine{00285\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classdef-\/>words,\ classdef-\/>weights,}
\DoxyCodeLine{00286\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classdef-\/>n\_words)\ <\ 0)\ \{}
\DoxyCodeLine{00287\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00288\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00289\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}Added\ class\ \%s\ containing\ \%d\ words\(\backslash\)n"{}},}
\DoxyCodeLine{00290\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ str,\ classdef-\/>n\_words);}
\DoxyCodeLine{00291\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00292\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (strcmp(str,\ \textcolor{stringliteral}{"{}\}"{}})\ !=\ 0)\ \{}
\DoxyCodeLine{00293\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Unexpected\ EOF\ in\ \%s\(\backslash\)n"{}},\ lmctlfile);}
\DoxyCodeLine{00294\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00295\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00296\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fscanf(ctlfp,\ \textcolor{stringliteral}{"{}\%1023s"{}},\ str)\ !=\ 1)}
\DoxyCodeLine{00297\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ str[0]\ =\ \textcolor{charliteral}{'\(\backslash\)0'};}
\DoxyCodeLine{00298\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00299\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00300\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00301\ \ \ \ \ \ \ \ \ \ \ \ \ str[0]\ =\ \textcolor{charliteral}{'\(\backslash\)0'};}
\DoxyCodeLine{00302\ \ \ \ \ \}}
\DoxyCodeLine{00303\ \ \ \ \ fclose(ctlfp);}
\DoxyCodeLine{00304\ }
\DoxyCodeLine{00305\ \ \ \ \ \textcolor{comment}{/*\ Now\ construct\ arrays\ out\ of\ lms\ and\ lmnames,\ and\ build\ an}}
\DoxyCodeLine{00306\ \textcolor{comment}{\ \ \ \ \ *\ ngram\_model\_set.\ */}}
\DoxyCodeLine{00307\ \ \ \ \ lms\ =\ glist\_reverse(lms);}
\DoxyCodeLine{00308\ \ \ \ \ lmnames\ =\ glist\_reverse(lmnames);}
\DoxyCodeLine{00309\ \ \ \ \ \{}
\DoxyCodeLine{00310\ \ \ \ \ \ \ \ \ int32\ n\_models;}
\DoxyCodeLine{00311\ \ \ \ \ \ \ \ \ ngram\_model\_t\ **lm\_array;}
\DoxyCodeLine{00312\ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{char}\ **name\_array;}
\DoxyCodeLine{00313\ \ \ \ \ \ \ \ \ gnode\_t\ *lm\_node,\ *name\_node;}
\DoxyCodeLine{00314\ \ \ \ \ \ \ \ \ int32\ i;}
\DoxyCodeLine{00315\ }
\DoxyCodeLine{00316\ \ \ \ \ \ \ \ \ n\_models\ =\ glist\_count(lms);}
\DoxyCodeLine{00317\ \ \ \ \ \ \ \ \ lm\_array\ =\ ckd\_calloc(n\_models,\ \textcolor{keyword}{sizeof}(*lm\_array));}
\DoxyCodeLine{00318\ \ \ \ \ \ \ \ \ name\_array\ =\ ckd\_calloc(n\_models,\ \textcolor{keyword}{sizeof}(*name\_array));}
\DoxyCodeLine{00319\ \ \ \ \ \ \ \ \ lm\_node\ =\ lms;}
\DoxyCodeLine{00320\ \ \ \ \ \ \ \ \ name\_node\ =\ lmnames;}
\DoxyCodeLine{00321\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ n\_models;\ ++i)\ \{}
\DoxyCodeLine{00322\ \ \ \ \ \ \ \ \ \ \ \ \ lm\_array[i]\ =\ gnode\_ptr(lm\_node);}
\DoxyCodeLine{00323\ \ \ \ \ \ \ \ \ \ \ \ \ name\_array[i]\ =\ gnode\_ptr(name\_node);}
\DoxyCodeLine{00324\ \ \ \ \ \ \ \ \ \ \ \ \ lm\_node\ =\ gnode\_next(lm\_node);}
\DoxyCodeLine{00325\ \ \ \ \ \ \ \ \ \ \ \ \ name\_node\ =\ gnode\_next(name\_node);}
\DoxyCodeLine{00326\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00327\ \ \ \ \ \ \ \ \ set\ =\ ngram\_model\_set\_init(config,\ lm\_array,\ name\_array,}
\DoxyCodeLine{00328\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ NULL,\ n\_models);}
\DoxyCodeLine{00329\ \ \ \ \ \ \ \ \ ckd\_free(lm\_array);}
\DoxyCodeLine{00330\ \ \ \ \ \ \ \ \ ckd\_free(name\_array);}
\DoxyCodeLine{00331\ \ \ \ \ \}}
\DoxyCodeLine{00332\ error\_out:}
\DoxyCodeLine{00333\ \ \ \ \ \{}
\DoxyCodeLine{00334\ \ \ \ \ \ \ \ \ gnode\_t\ *gn;}
\DoxyCodeLine{00335\ \ \ \ \ \ \ \ \ glist\_t\ hlist;}
\DoxyCodeLine{00336\ }
\DoxyCodeLine{00337\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (set\ ==\ NULL)\ \{}
\DoxyCodeLine{00338\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (gn\ =\ lms;\ gn;\ gn\ =\ gnode\_next(gn))\ \{}
\DoxyCodeLine{00339\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_model\_free(gnode\_ptr(gn));}
\DoxyCodeLine{00340\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00341\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00342\ \ \ \ \ \ \ \ \ glist\_free(lms);}
\DoxyCodeLine{00343\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (gn\ =\ lmnames;\ gn;\ gn\ =\ gnode\_next(gn))\ \{}
\DoxyCodeLine{00344\ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_free(gnode\_ptr(gn));}
\DoxyCodeLine{00345\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00346\ \ \ \ \ \ \ \ \ glist\_free(lmnames);}
\DoxyCodeLine{00347\ \ \ \ \ \ \ \ \ hlist\ =\ hash\_table\_tolist(classes,\ NULL);}
\DoxyCodeLine{00348\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (gn\ =\ hlist;\ gn;\ gn\ =\ gnode\_next(gn))\ \{}
\DoxyCodeLine{00349\ \ \ \ \ \ \ \ \ \ \ \ \ hash\_entry\_t\ *he\ =\ gnode\_ptr(gn);}
\DoxyCodeLine{00350\ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_free((\textcolor{keywordtype}{char}\ *)he-\/>key);}
\DoxyCodeLine{00351\ \ \ \ \ \ \ \ \ \ \ \ \ classdef\_free(he-\/>val);}
\DoxyCodeLine{00352\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00353\ \ \ \ \ \ \ \ \ glist\_free(hlist);}
\DoxyCodeLine{00354\ \ \ \ \ \ \ \ \ hash\_table\_free(classes);}
\DoxyCodeLine{00355\ \ \ \ \ \ \ \ \ ckd\_free(basedir);}
\DoxyCodeLine{00356\ \ \ \ \ \}}
\DoxyCodeLine{00357\ \ \ \ \ \textcolor{keywordflow}{return}\ set;}
\DoxyCodeLine{00358\ \}}
\DoxyCodeLine{00359\ }
\DoxyCodeLine{00360\ int32}
\DoxyCodeLine{00361\ ngram\_model\_set\_count(ngram\_model\_t\ *base)}
\DoxyCodeLine{00362\ \{}
\DoxyCodeLine{00363\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00364\ \ \ \ \ \textcolor{keywordflow}{return}\ set-\/>n\_models;}
\DoxyCodeLine{00365\ \}}
\DoxyCodeLine{00366\ }
\DoxyCodeLine{00367\ ngram\_model\_set\_iter\_t\ *}
\DoxyCodeLine{00368\ ngram\_model\_set\_iter(ngram\_model\_t\ *base)}
\DoxyCodeLine{00369\ \{}
\DoxyCodeLine{00370\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00371\ \ \ \ \ ngram\_model\_set\_iter\_t\ *itor;}
\DoxyCodeLine{00372\ }
\DoxyCodeLine{00373\ \ \ \ \ \textcolor{keywordflow}{if}\ (set\ ==\ NULL\ ||\ set-\/>n\_models\ ==\ 0)}
\DoxyCodeLine{00374\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00375\ \ \ \ \ itor\ =\ ckd\_calloc(1,\ \textcolor{keyword}{sizeof}(*itor));}
\DoxyCodeLine{00376\ \ \ \ \ itor-\/>set\ =\ set;}
\DoxyCodeLine{00377\ \ \ \ \ \textcolor{keywordflow}{return}\ itor;}
\DoxyCodeLine{00378\ \}}
\DoxyCodeLine{00379\ }
\DoxyCodeLine{00380\ ngram\_model\_set\_iter\_t\ *}
\DoxyCodeLine{00381\ ngram\_model\_set\_iter\_next(ngram\_model\_set\_iter\_t\ *itor)}
\DoxyCodeLine{00382\ \{}
\DoxyCodeLine{00383\ \ \ \ \ \textcolor{keywordflow}{if}\ (++itor-\/>cur\ ==\ itor-\/>set-\/>n\_models)\ \{}
\DoxyCodeLine{00384\ \ \ \ \ \ \ \ \ ngram\_model\_set\_iter\_free(itor);}
\DoxyCodeLine{00385\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00386\ \ \ \ \ \}}
\DoxyCodeLine{00387\ \ \ \ \ \textcolor{keywordflow}{return}\ itor;}
\DoxyCodeLine{00388\ \}}
\DoxyCodeLine{00389\ }
\DoxyCodeLine{00390\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00391\ ngram\_model\_set\_iter\_free(ngram\_model\_set\_iter\_t\ *itor)}
\DoxyCodeLine{00392\ \{}
\DoxyCodeLine{00393\ \ \ \ \ ckd\_free(itor);}
\DoxyCodeLine{00394\ \}}
\DoxyCodeLine{00395\ }
\DoxyCodeLine{00396\ ngram\_model\_t\ *}
\DoxyCodeLine{00397\ ngram\_model\_set\_iter\_model(ngram\_model\_set\_iter\_t\ *itor,}
\DoxyCodeLine{00398\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{char}\ \textcolor{keyword}{const}\ **lmname)}
\DoxyCodeLine{00399\ \{}
\DoxyCodeLine{00400\ \ \ \ \ \textcolor{keywordflow}{if}\ (lmname)\ *lmname\ =\ itor-\/>set-\/>names[itor-\/>cur];}
\DoxyCodeLine{00401\ \ \ \ \ \textcolor{keywordflow}{return}\ itor-\/>set-\/>lms[itor-\/>cur];}
\DoxyCodeLine{00402\ \}}
\DoxyCodeLine{00403\ }
\DoxyCodeLine{00404\ ngram\_model\_t\ *}
\DoxyCodeLine{00405\ ngram\_model\_set\_lookup(ngram\_model\_t\ *base,}
\DoxyCodeLine{00406\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *name)}
\DoxyCodeLine{00407\ \{}
\DoxyCodeLine{00408\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00409\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00410\ }
\DoxyCodeLine{00411\ \ \ \ \ \textcolor{keywordflow}{if}\ (name\ ==\ NULL)\ \{}
\DoxyCodeLine{00412\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (set-\/>cur\ ==\ -\/1)}
\DoxyCodeLine{00413\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00414\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00415\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ set-\/>lms[set-\/>cur];}
\DoxyCodeLine{00416\ \ \ \ \ \}}
\DoxyCodeLine{00417\ }
\DoxyCodeLine{00418\ \ \ \ \ \textcolor{comment}{/*\ There\ probably\ won't\ be\ very\ many\ submodels.\ */}}
\DoxyCodeLine{00419\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ set-\/>n\_models;\ ++i)}
\DoxyCodeLine{00420\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (0\ ==\ strcmp(set-\/>names[i],\ name))}
\DoxyCodeLine{00421\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00422\ \ \ \ \ \textcolor{keywordflow}{if}\ (i\ ==\ set-\/>n\_models)}
\DoxyCodeLine{00423\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00424\ \ \ \ \ \textcolor{keywordflow}{return}\ set-\/>lms[i];}
\DoxyCodeLine{00425\ \}}
\DoxyCodeLine{00426\ }
\DoxyCodeLine{00427\ ngram\_model\_t\ *}
\DoxyCodeLine{00428\ ngram\_model\_set\_select(ngram\_model\_t\ *base,}
\DoxyCodeLine{00429\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *name)}
\DoxyCodeLine{00430\ \{}
\DoxyCodeLine{00431\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00432\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00433\ }
\DoxyCodeLine{00434\ \ \ \ \ \textcolor{comment}{/*\ There\ probably\ won't\ be\ very\ many\ submodels.\ */}}
\DoxyCodeLine{00435\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ set-\/>n\_models;\ ++i)}
\DoxyCodeLine{00436\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (0\ ==\ strcmp(set-\/>names[i],\ name))}
\DoxyCodeLine{00437\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00438\ \ \ \ \ \textcolor{keywordflow}{if}\ (i\ ==\ set-\/>n\_models)}
\DoxyCodeLine{00439\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00440\ \ \ \ \ set-\/>cur\ =\ i;}
\DoxyCodeLine{00441\ \ \ \ \ \textcolor{keywordflow}{return}\ set-\/>lms[set-\/>cur];}
\DoxyCodeLine{00442\ \}}
\DoxyCodeLine{00443\ }
\DoxyCodeLine{00444\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *}
\DoxyCodeLine{00445\ ngram\_model\_set\_current(ngram\_model\_t\ *base)}
\DoxyCodeLine{00446\ \{}
\DoxyCodeLine{00447\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00448\ }
\DoxyCodeLine{00449\ \ \ \ \ \textcolor{keywordflow}{if}\ (set-\/>cur\ ==\ -\/1)}
\DoxyCodeLine{00450\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00451\ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00452\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ set-\/>names[set-\/>cur];}
\DoxyCodeLine{00453\ \}}
\DoxyCodeLine{00454\ }
\DoxyCodeLine{00455\ int32}
\DoxyCodeLine{00456\ ngram\_model\_set\_current\_wid(ngram\_model\_t\ *base,}
\DoxyCodeLine{00457\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ set\_wid)}
\DoxyCodeLine{00458\ \{}
\DoxyCodeLine{00459\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00460\ }
\DoxyCodeLine{00461\ \ \ \ \ \textcolor{keywordflow}{if}\ (set-\/>cur\ ==\ -\/1\ ||\ set\_wid\ >=\ base-\/>n\_words)}
\DoxyCodeLine{00462\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NGRAM\_INVALID\_WID;}
\DoxyCodeLine{00463\ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00464\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ set-\/>widmap[set\_wid][set-\/>cur];}
\DoxyCodeLine{00465\ \}}
\DoxyCodeLine{00466\ }
\DoxyCodeLine{00467\ int32}
\DoxyCodeLine{00468\ ngram\_model\_set\_known\_wid(ngram\_model\_t\ *base,}
\DoxyCodeLine{00469\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ set\_wid)}
\DoxyCodeLine{00470\ \{}
\DoxyCodeLine{00471\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00472\ }
\DoxyCodeLine{00473\ \ \ \ \ \textcolor{keywordflow}{if}\ (set\_wid\ >=\ base-\/>n\_words)}
\DoxyCodeLine{00474\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ FALSE;}
\DoxyCodeLine{00475\ \ \ \ \ \textcolor{keywordflow}{else}\ \textcolor{keywordflow}{if}\ (set-\/>cur\ ==\ -\/1)\ \{}
\DoxyCodeLine{00476\ \ \ \ \ \ \ \ \ int32\ i;}
\DoxyCodeLine{00477\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ set-\/>n\_models;\ ++i)\ \{}
\DoxyCodeLine{00478\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (set-\/>widmap[set\_wid][i]\ !=\ ngram\_unknown\_wid(set-\/>lms[i]))}
\DoxyCodeLine{00479\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ TRUE;}
\DoxyCodeLine{00480\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00481\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ FALSE;}
\DoxyCodeLine{00482\ \ \ \ \ \}}
\DoxyCodeLine{00483\ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00484\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ (set-\/>widmap[set\_wid][set-\/>cur]}
\DoxyCodeLine{00485\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ !=\ ngram\_unknown\_wid(set-\/>lms[set-\/>cur]));}
\DoxyCodeLine{00486\ \}}
\DoxyCodeLine{00487\ }
\DoxyCodeLine{00488\ ngram\_model\_t\ *}
\DoxyCodeLine{00489\ ngram\_model\_set\_interp(ngram\_model\_t\ *base,}
\DoxyCodeLine{00490\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ **names,}
\DoxyCodeLine{00491\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ float32\ *weights)}
\DoxyCodeLine{00492\ \{}
\DoxyCodeLine{00493\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00494\ }
\DoxyCodeLine{00495\ \ \ \ \ \textcolor{comment}{/*\ If\ we\ have\ a\ set\ of\ weights\ here,\ then\ set\ them.\ */}}
\DoxyCodeLine{00496\ \ \ \ \ \textcolor{keywordflow}{if}\ (names\ \&\&\ weights)\ \{}
\DoxyCodeLine{00497\ \ \ \ \ \ \ \ \ int32\ i,\ j;}
\DoxyCodeLine{00498\ }
\DoxyCodeLine{00499\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ We\ hope\ there\ aren't\ many\ models.\ */}}
\DoxyCodeLine{00500\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ set-\/>n\_models;\ ++i)\ \{}
\DoxyCodeLine{00501\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (j\ =\ 0;\ j\ <\ set-\/>n\_models;\ ++j)}
\DoxyCodeLine{00502\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (0\ ==\ strcmp(names[i],\ set-\/>names[j]))}
\DoxyCodeLine{00503\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00504\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (j\ ==\ set-\/>n\_models)\ \{}
\DoxyCodeLine{00505\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Unknown\ LM\ name\ \%s\(\backslash\)n"{}},\ names[i]);}
\DoxyCodeLine{00506\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00507\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00508\ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>lweights[j]\ =\ logmath\_log(base-\/>lmath,\ weights[i]);}
\DoxyCodeLine{00509\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00510\ \ \ \ \ \}}
\DoxyCodeLine{00511\ \ \ \ \ \textcolor{keywordflow}{else}\ \textcolor{keywordflow}{if}\ (weights)\ \{}
\DoxyCodeLine{00512\ \ \ \ \ \ \ \ \ memcpy(set-\/>lweights,\ weights,\ set-\/>n\_models\ *\ \textcolor{keyword}{sizeof}(*set-\/>lweights));}
\DoxyCodeLine{00513\ \ \ \ \ \}}
\DoxyCodeLine{00514\ \ \ \ \ \textcolor{comment}{/*\ Otherwise\ just\ enable\ existing\ weights.\ */}}
\DoxyCodeLine{00515\ \ \ \ \ set-\/>cur\ =\ -\/1;}
\DoxyCodeLine{00516\ \ \ \ \ \textcolor{keywordflow}{return}\ base;}
\DoxyCodeLine{00517\ \}}
\DoxyCodeLine{00518\ }
\DoxyCodeLine{00519\ ngram\_model\_t\ *}
\DoxyCodeLine{00520\ ngram\_model\_set\_add(ngram\_model\_t\ *base,}
\DoxyCodeLine{00521\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_model\_t\ *model,}
\DoxyCodeLine{00522\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *name,}
\DoxyCodeLine{00523\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ float32\ weight,}
\DoxyCodeLine{00524\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{int}\ reuse\_widmap)}
\DoxyCodeLine{00525\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }
\DoxyCodeLine{00526\ \{}
\DoxyCodeLine{00527\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00528\ \ \ \ \ float32\ fprob;}
\DoxyCodeLine{00529\ \ \ \ \ int32\ scale,\ i;}
\DoxyCodeLine{00530\ }
\DoxyCodeLine{00531\ \ \ \ \ \textcolor{comment}{/*\ Add\ it\ to\ the\ array\ of\ lms.\ */}}
\DoxyCodeLine{00532\ \ \ \ \ ++set-\/>n\_models;}
\DoxyCodeLine{00533\ \ \ \ \ set-\/>lms\ =\ ckd\_realloc(set-\/>lms,\ set-\/>n\_models\ *\ \textcolor{keyword}{sizeof}(*set-\/>lms));}
\DoxyCodeLine{00534\ \ \ \ \ set-\/>lms[set-\/>n\_models\ -\/\ 1]\ =\ model;}
\DoxyCodeLine{00535\ \ \ \ \ set-\/>names\ =\ ckd\_realloc(set-\/>names,\ set-\/>n\_models\ *\ \textcolor{keyword}{sizeof}(*set-\/>names));}
\DoxyCodeLine{00536\ \ \ \ \ set-\/>names[set-\/>n\_models\ -\/\ 1]\ =\ ckd\_salloc(name);}
\DoxyCodeLine{00537\ \ \ \ \ \textcolor{comment}{/*\ Expand\ the\ history\ mapping\ table\ if\ necessary.\ */}}
\DoxyCodeLine{00538\ \ \ \ \ \textcolor{keywordflow}{if}\ (model-\/>n\ >\ base-\/>n)\ \{}
\DoxyCodeLine{00539\ \ \ \ \ \ \ \ \ base-\/>n\ =\ model-\/>n;}
\DoxyCodeLine{00540\ \ \ \ \ \ \ \ \ set-\/>maphist\ =\ ckd\_realloc(set-\/>maphist,}
\DoxyCodeLine{00541\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (model-\/>n\ -\/\ 1)\ *\ \textcolor{keyword}{sizeof}(*set-\/>maphist));}
\DoxyCodeLine{00542\ \ \ \ \ \}}
\DoxyCodeLine{00543\ }
\DoxyCodeLine{00544\ \ \ \ \ \textcolor{comment}{/*\ Renormalize\ the\ interpolation\ weights.\ */}}
\DoxyCodeLine{00545\ \ \ \ \ fprob\ =\ weight\ *\ 1.0\ /\ set-\/>n\_models;}
\DoxyCodeLine{00546\ \ \ \ \ set-\/>lweights\ =\ ckd\_realloc(set-\/>lweights,}
\DoxyCodeLine{00547\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>n\_models\ *\ \textcolor{keyword}{sizeof}(*set-\/>lweights));}
\DoxyCodeLine{00548\ \ \ \ \ set-\/>lweights[set-\/>n\_models\ -\/\ 1]\ =\ logmath\_log(base-\/>lmath,\ fprob);}
\DoxyCodeLine{00549\ \ \ \ \ \textcolor{comment}{/*\ Now\ normalize\ everything\ else\ to\ fit\ it\ in.\ \ This\ is}}
\DoxyCodeLine{00550\ \textcolor{comment}{\ \ \ \ \ *\ accomplished\ by\ simply\ scaling\ all\ the\ other\ probabilities}}
\DoxyCodeLine{00551\ \textcolor{comment}{\ \ \ \ \ *\ by\ (1-\/fprob).\ */}}
\DoxyCodeLine{00552\ \ \ \ \ scale\ =\ logmath\_log(base-\/>lmath,\ 1.0\ -\/\ fprob);}
\DoxyCodeLine{00553\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ set-\/>n\_models\ -\/\ 1;\ ++i)}
\DoxyCodeLine{00554\ \ \ \ \ \ \ \ \ set-\/>lweights[i]\ +=\ scale;}
\DoxyCodeLine{00555\ }
\DoxyCodeLine{00556\ \ \ \ \ \textcolor{comment}{/*\ Reuse\ the\ old\ word\ ID\ mapping\ if\ requested.\ */}}
\DoxyCodeLine{00557\ \ \ \ \ \textcolor{keywordflow}{if}\ (reuse\_widmap)\ \{}
\DoxyCodeLine{00558\ \ \ \ \ \ \ \ \ int32\ **new\_widmap;}
\DoxyCodeLine{00559\ }
\DoxyCodeLine{00560\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Tack\ another\ column\ onto\ the\ widmap\ array.\ */}}
\DoxyCodeLine{00561\ \ \ \ \ \ \ \ \ new\_widmap\ =\ (int32\ **)ckd\_calloc\_2d(base-\/>n\_words,\ set-\/>n\_models,}
\DoxyCodeLine{00562\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ sizeof\ (**new\_widmap));}
\DoxyCodeLine{00563\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ base-\/>n\_words;\ ++i)\ \{}
\DoxyCodeLine{00564\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Copy\ all\ the\ existing\ mappings.\ */}}
\DoxyCodeLine{00565\ \ \ \ \ \ \ \ \ \ \ \ \ memcpy(new\_widmap[i],\ set-\/>widmap[i],}
\DoxyCodeLine{00566\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (set-\/>n\_models\ -\/\ 1)\ *\ \textcolor{keyword}{sizeof}(**new\_widmap));}
\DoxyCodeLine{00567\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Create\ the\ new\ mapping.\ */}}
\DoxyCodeLine{00568\ \ \ \ \ \ \ \ \ \ \ \ \ new\_widmap[i][set-\/>n\_models-\/1]\ =\ ngram\_wid(model,\ base-\/>word\_str[i]);}
\DoxyCodeLine{00569\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00570\ \ \ \ \ \ \ \ \ ckd\_free\_2d((\textcolor{keywordtype}{void}\ **)set-\/>widmap);}
\DoxyCodeLine{00571\ \ \ \ \ \ \ \ \ set-\/>widmap\ =\ new\_widmap;}
\DoxyCodeLine{00572\ \ \ \ \ \}}
\DoxyCodeLine{00573\ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00574\ \ \ \ \ \ \ \ \ build\_widmap(base,\ base-\/>lmath,\ base-\/>n);}
\DoxyCodeLine{00575\ \ \ \ \ \}}
\DoxyCodeLine{00576\ \ \ \ \ \textcolor{keywordflow}{return}\ model;}
\DoxyCodeLine{00577\ \}}
\DoxyCodeLine{00578\ }
\DoxyCodeLine{00579\ ngram\_model\_t\ *}
\DoxyCodeLine{00580\ ngram\_model\_set\_remove(ngram\_model\_t\ *base,}
\DoxyCodeLine{00581\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *name,}
\DoxyCodeLine{00582\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{int}\ reuse\_widmap)}
\DoxyCodeLine{00583\ \{}
\DoxyCodeLine{00584\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00585\ \ \ \ \ ngram\_model\_t\ *submodel;}
\DoxyCodeLine{00586\ \ \ \ \ int32\ lmidx,\ scale,\ n,\ i;}
\DoxyCodeLine{00587\ \ \ \ \ float32\ fprob;}
\DoxyCodeLine{00588\ }
\DoxyCodeLine{00589\ \ \ \ \ \textcolor{keywordflow}{for}\ (lmidx\ =\ 0;\ lmidx\ <\ set-\/>n\_models;\ ++lmidx)}
\DoxyCodeLine{00590\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (0\ ==\ strcmp(name,\ set-\/>names[lmidx]))}
\DoxyCodeLine{00591\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00592\ \ \ \ \ \textcolor{keywordflow}{if}\ (lmidx\ ==\ set-\/>n\_models)}
\DoxyCodeLine{00593\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00594\ \ \ \ \ submodel\ =\ set-\/>lms[lmidx];}
\DoxyCodeLine{00595\ }
\DoxyCodeLine{00596\ \ \ \ \ \textcolor{comment}{/*\ Renormalize\ the\ interpolation\ weights\ by\ scaling\ them\ by}}
\DoxyCodeLine{00597\ \textcolor{comment}{\ \ \ \ \ *\ 1/(1-\/fprob)\ */}}
\DoxyCodeLine{00598\ \ \ \ \ fprob\ =\ logmath\_exp(base-\/>lmath,\ set-\/>lweights[lmidx]);}
\DoxyCodeLine{00599\ \ \ \ \ scale\ =\ logmath\_log(base-\/>lmath,\ 1.0\ -\/\ fprob);}
\DoxyCodeLine{00600\ }
\DoxyCodeLine{00601\ \ \ \ \ \textcolor{comment}{/*\ Remove\ it\ from\ the\ array\ of\ lms,\ renormalize\ remaining\ weights,}}
\DoxyCodeLine{00602\ \textcolor{comment}{\ \ \ \ \ *\ and\ recalcluate\ n.\ */}}
\DoxyCodeLine{00603\ \ \ \ \ -\/-\/set-\/>n\_models;}
\DoxyCodeLine{00604\ \ \ \ \ n\ =\ 0;}
\DoxyCodeLine{00605\ \ \ \ \ ckd\_free(set-\/>names[lmidx]);}
\DoxyCodeLine{00606\ \ \ \ \ set-\/>names[lmidx]\ =\ NULL;}
\DoxyCodeLine{00607\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ set-\/>n\_models;\ ++i)\ \{}
\DoxyCodeLine{00608\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (i\ >=\ lmidx)\ \{}
\DoxyCodeLine{00609\ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>lms[i]\ =\ set-\/>lms[i+1];}
\DoxyCodeLine{00610\ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>names[i]\ =\ set-\/>names[i+1];}
\DoxyCodeLine{00611\ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>lweights[i]\ =\ set-\/>lweights[i+1];}
\DoxyCodeLine{00612\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00613\ \ \ \ \ \ \ \ \ set-\/>lweights[i]\ -\/=\ scale;}
\DoxyCodeLine{00614\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (set-\/>lms[i]-\/>n\ >\ n)}
\DoxyCodeLine{00615\ \ \ \ \ \ \ \ \ \ \ \ \ n\ =\ set-\/>lms[i]-\/>n;}
\DoxyCodeLine{00616\ \ \ \ \ \}}
\DoxyCodeLine{00617\ \ \ \ \ \textcolor{comment}{/*\ There's\ no\ need\ to\ shrink\ these\ arrays.\ */}}
\DoxyCodeLine{00618\ \ \ \ \ set-\/>lms[set-\/>n\_models]\ =\ NULL;}
\DoxyCodeLine{00619\ \ \ \ \ set-\/>lweights[set-\/>n\_models]\ =\ base-\/>log\_zero;}
\DoxyCodeLine{00620\ \ \ \ \ \textcolor{comment}{/*\ No\ need\ to\ shrink\ maphist\ either.\ */}}
\DoxyCodeLine{00621\ }
\DoxyCodeLine{00622\ \ \ \ \ \textcolor{comment}{/*\ Reuse\ the\ existing\ word\ ID\ mapping\ if\ requested.\ */}}
\DoxyCodeLine{00623\ \ \ \ \ \textcolor{keywordflow}{if}\ (reuse\_widmap)\ \{}
\DoxyCodeLine{00624\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Just\ go\ through\ and\ shrink\ each\ row.\ */}}
\DoxyCodeLine{00625\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ base-\/>n\_words;\ ++i)\ \{}
\DoxyCodeLine{00626\ \ \ \ \ \ \ \ \ \ \ \ \ memmove(set-\/>widmap[i]\ +\ lmidx,\ set-\/>widmap[i]\ +\ lmidx\ +\ 1,}
\DoxyCodeLine{00627\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (set-\/>n\_models\ -\/\ lmidx)\ *\ \textcolor{keyword}{sizeof}(**set-\/>widmap));}
\DoxyCodeLine{00628\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00629\ \ \ \ \ \}}
\DoxyCodeLine{00630\ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00631\ \ \ \ \ \ \ \ \ build\_widmap(base,\ base-\/>lmath,\ n);}
\DoxyCodeLine{00632\ \ \ \ \ \}}
\DoxyCodeLine{00633\ \ \ \ \ \textcolor{keywordflow}{return}\ submodel;}
\DoxyCodeLine{00634\ \}}
\DoxyCodeLine{00635\ }
\DoxyCodeLine{00636\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00637\ ngram\_model\_set\_map\_words(ngram\_model\_t\ *base,}
\DoxyCodeLine{00638\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ **words,}
\DoxyCodeLine{00639\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ n\_words)}
\DoxyCodeLine{00640\ \{}
\DoxyCodeLine{00641\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00642\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00643\ }
\DoxyCodeLine{00644\ \ \ \ \ \textcolor{comment}{/*\ Recreate\ the\ word\ mapping.\ */}}
\DoxyCodeLine{00645\ \ \ \ \ \textcolor{keywordflow}{if}\ (base-\/>writable)\ \{}
\DoxyCodeLine{00646\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ base-\/>n\_words;\ ++i)\ \{}
\DoxyCodeLine{00647\ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_free(base-\/>word\_str[i]);}
\DoxyCodeLine{00648\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00649\ \ \ \ \ \}}
\DoxyCodeLine{00650\ \ \ \ \ ckd\_free(base-\/>word\_str);}
\DoxyCodeLine{00651\ \ \ \ \ ckd\_free\_2d((\textcolor{keywordtype}{void}\ **)set-\/>widmap);}
\DoxyCodeLine{00652\ \ \ \ \ base-\/>writable\ =\ TRUE;}
\DoxyCodeLine{00653\ \ \ \ \ base-\/>n\_words\ =\ base-\/>n\_1g\_alloc\ =\ n\_words;}
\DoxyCodeLine{00654\ \ \ \ \ base-\/>word\_str\ =\ ckd\_calloc(n\_words,\ \textcolor{keyword}{sizeof}(*base-\/>word\_str));}
\DoxyCodeLine{00655\ \ \ \ \ set-\/>widmap\ =\ (int32\ **)ckd\_calloc\_2d(n\_words,\ set-\/>n\_models,\ \textcolor{keyword}{sizeof}(**set-\/>widmap));}
\DoxyCodeLine{00656\ \ \ \ \ hash\_table\_empty(base-\/>wid);}
\DoxyCodeLine{00657\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ n\_words;\ ++i)\ \{}
\DoxyCodeLine{00658\ \ \ \ \ \ \ \ \ int32\ j;}
\DoxyCodeLine{00659\ \ \ \ \ \ \ \ \ base-\/>word\_str[i]\ =\ ckd\_salloc(words[i]);}
\DoxyCodeLine{00660\ \ \ \ \ \ \ \ \ (void)hash\_table\_enter\_int32(base-\/>wid,\ base-\/>word\_str[i],\ i);}
\DoxyCodeLine{00661\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (j\ =\ 0;\ j\ <\ set-\/>n\_models;\ ++j)\ \{}
\DoxyCodeLine{00662\ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>widmap[i][j]\ =\ ngram\_wid(set-\/>lms[j],\ base-\/>word\_str[i]);}
\DoxyCodeLine{00663\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00664\ \ \ \ \ \}}
\DoxyCodeLine{00665\ \}}
\DoxyCodeLine{00666\ }
\DoxyCodeLine{00667\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{int}}
\DoxyCodeLine{00668\ ngram\_model\_set\_apply\_weights(ngram\_model\_t\ *base,\ float32\ lw,}
\DoxyCodeLine{00669\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ float32\ wip,\ float32\ uw)}
\DoxyCodeLine{00670\ \{}
\DoxyCodeLine{00671\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00672\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00673\ }
\DoxyCodeLine{00674\ \ \ \ \ \textcolor{comment}{/*\ Apply\ weights\ to\ each\ sub-\/model.\ */}}
\DoxyCodeLine{00675\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ set-\/>n\_models;\ ++i)}
\DoxyCodeLine{00676\ \ \ \ \ \ \ \ \ ngram\_model\_apply\_weights(set-\/>lms[i],\ lw,\ wip,\ uw);}
\DoxyCodeLine{00677\ \ \ \ \ \textcolor{keywordflow}{return}\ 0;}
\DoxyCodeLine{00678\ \}}
\DoxyCodeLine{00679\ }
\DoxyCodeLine{00680\ \textcolor{keyword}{static}\ int32}
\DoxyCodeLine{00681\ ngram\_model\_set\_score(ngram\_model\_t\ *base,\ int32\ wid,}
\DoxyCodeLine{00682\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ *history,\ int32\ n\_hist,}
\DoxyCodeLine{00683\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ *n\_used)}
\DoxyCodeLine{00684\ \{}
\DoxyCodeLine{00685\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00686\ \ \ \ \ int32\ mapwid;}
\DoxyCodeLine{00687\ \ \ \ \ int32\ score;}
\DoxyCodeLine{00688\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00689\ }
\DoxyCodeLine{00690\ \ \ \ \ \textcolor{comment}{/*\ Truncate\ the\ history.\ */}}
\DoxyCodeLine{00691\ \ \ \ \ \textcolor{keywordflow}{if}\ (n\_hist\ >\ base-\/>n\ -\/\ 1)}
\DoxyCodeLine{00692\ \ \ \ \ \ \ \ \ n\_hist\ =\ base-\/>n\ -\/\ 1;}
\DoxyCodeLine{00693\ }
\DoxyCodeLine{00694\ \ \ \ \ \textcolor{comment}{/*\ Interpolate\ if\ there\ is\ no\ current.\ */}}
\DoxyCodeLine{00695\ \ \ \ \ \textcolor{keywordflow}{if}\ (set-\/>cur\ ==\ -\/1)\ \{}
\DoxyCodeLine{00696\ \ \ \ \ \ \ \ \ score\ =\ base-\/>log\_zero;}
\DoxyCodeLine{00697\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ set-\/>n\_models;\ ++i)\ \{}
\DoxyCodeLine{00698\ \ \ \ \ \ \ \ \ \ \ \ \ int32\ j;}
\DoxyCodeLine{00699\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Map\ word\ and\ history\ IDs\ for\ each\ model.\ */}}
\DoxyCodeLine{00700\ \ \ \ \ \ \ \ \ \ \ \ \ mapwid\ =\ set-\/>widmap[wid][i];}
\DoxyCodeLine{00701\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (j\ =\ 0;\ j\ <\ n\_hist;\ ++j)\ \{}
\DoxyCodeLine{00702\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (history[j]\ ==\ NGRAM\_INVALID\_WID)}
\DoxyCodeLine{00703\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>maphist[j]\ =\ NGRAM\_INVALID\_WID;}
\DoxyCodeLine{00704\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00705\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>maphist[j]\ =\ set-\/>widmap[history[j]][i];}
\DoxyCodeLine{00706\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00707\ \ \ \ \ \ \ \ \ \ \ \ \ score\ =\ logmath\_add(base-\/>lmath,\ score,}
\DoxyCodeLine{00708\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>lweights[i]\ +\ }
\DoxyCodeLine{00709\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_ng\_score(set-\/>lms[i],}
\DoxyCodeLine{00710\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ mapwid,\ set-\/>maphist,\ n\_hist,\ n\_used));}
\DoxyCodeLine{00711\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00712\ \ \ \ \ \}}
\DoxyCodeLine{00713\ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00714\ \ \ \ \ \ \ \ \ int32\ j;}
\DoxyCodeLine{00715\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Map\ word\ and\ history\ IDs\ (FIXME:\ do\ this\ in\ a\ function?)\ */}}
\DoxyCodeLine{00716\ \ \ \ \ \ \ \ \ mapwid\ =\ set-\/>widmap[wid][set-\/>cur];}
\DoxyCodeLine{00717\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (j\ =\ 0;\ j\ <\ n\_hist;\ ++j)\ \{}
\DoxyCodeLine{00718\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (history[j]\ ==\ NGRAM\_INVALID\_WID)}
\DoxyCodeLine{00719\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>maphist[j]\ =\ NGRAM\_INVALID\_WID;}
\DoxyCodeLine{00720\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00721\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>maphist[j]\ =\ set-\/>widmap[history[j]][set-\/>cur];}
\DoxyCodeLine{00722\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00723\ \ \ \ \ \ \ \ \ score\ =\ ngram\_ng\_score(set-\/>lms[set-\/>cur],}
\DoxyCodeLine{00724\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ mapwid,\ set-\/>maphist,\ n\_hist,\ n\_used);}
\DoxyCodeLine{00725\ \ \ \ \ \}}
\DoxyCodeLine{00726\ }
\DoxyCodeLine{00727\ \ \ \ \ \textcolor{keywordflow}{return}\ score;}
\DoxyCodeLine{00728\ \}}
\DoxyCodeLine{00729\ }
\DoxyCodeLine{00730\ \textcolor{keyword}{static}\ int32}
\DoxyCodeLine{00731\ ngram\_model\_set\_raw\_score(ngram\_model\_t\ *base,\ int32\ wid,}
\DoxyCodeLine{00732\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ *history,\ int32\ n\_hist,}
\DoxyCodeLine{00733\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ *n\_used)}
\DoxyCodeLine{00734\ \{}
\DoxyCodeLine{00735\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00736\ \ \ \ \ int32\ mapwid;}
\DoxyCodeLine{00737\ \ \ \ \ int32\ score;}
\DoxyCodeLine{00738\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00739\ }
\DoxyCodeLine{00740\ \ \ \ \ \textcolor{comment}{/*\ Truncate\ the\ history.\ */}}
\DoxyCodeLine{00741\ \ \ \ \ \textcolor{keywordflow}{if}\ (n\_hist\ >\ base-\/>n\ -\/\ 1)}
\DoxyCodeLine{00742\ \ \ \ \ \ \ \ \ n\_hist\ =\ base-\/>n\ -\/\ 1;}
\DoxyCodeLine{00743\ }
\DoxyCodeLine{00744\ \ \ \ \ \textcolor{comment}{/*\ Interpolate\ if\ there\ is\ no\ current.\ */}}
\DoxyCodeLine{00745\ \ \ \ \ \textcolor{keywordflow}{if}\ (set-\/>cur\ ==\ -\/1)\ \{}
\DoxyCodeLine{00746\ \ \ \ \ \ \ \ \ score\ =\ base-\/>log\_zero;}
\DoxyCodeLine{00747\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ set-\/>n\_models;\ ++i)\ \{}
\DoxyCodeLine{00748\ \ \ \ \ \ \ \ \ \ \ \ \ int32\ j;}
\DoxyCodeLine{00749\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Map\ word\ and\ history\ IDs\ for\ each\ model.\ */}}
\DoxyCodeLine{00750\ \ \ \ \ \ \ \ \ \ \ \ \ mapwid\ =\ set-\/>widmap[wid][i];}
\DoxyCodeLine{00751\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (j\ =\ 0;\ j\ <\ n\_hist;\ ++j)\ \{}
\DoxyCodeLine{00752\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (history[j]\ ==\ NGRAM\_INVALID\_WID)}
\DoxyCodeLine{00753\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>maphist[j]\ =\ NGRAM\_INVALID\_WID;}
\DoxyCodeLine{00754\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00755\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>maphist[j]\ =\ set-\/>widmap[history[j]][i];}
\DoxyCodeLine{00756\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00757\ \ \ \ \ \ \ \ \ \ \ \ \ score\ =\ logmath\_add(base-\/>lmath,\ score,}
\DoxyCodeLine{00758\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>lweights[i]\ +\ }
\DoxyCodeLine{00759\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_ng\_prob(set-\/>lms[i],}
\DoxyCodeLine{00760\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ mapwid,\ set-\/>maphist,\ n\_hist,\ n\_used));}
\DoxyCodeLine{00761\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00762\ \ \ \ \ \}}
\DoxyCodeLine{00763\ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00764\ \ \ \ \ \ \ \ \ int32\ j;}
\DoxyCodeLine{00765\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Map\ word\ and\ history\ IDs\ (FIXME:\ do\ this\ in\ a\ function?)\ */}}
\DoxyCodeLine{00766\ \ \ \ \ \ \ \ \ mapwid\ =\ set-\/>widmap[wid][set-\/>cur];}
\DoxyCodeLine{00767\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (j\ =\ 0;\ j\ <\ n\_hist;\ ++j)\ \{}
\DoxyCodeLine{00768\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (history[j]\ ==\ NGRAM\_INVALID\_WID)}
\DoxyCodeLine{00769\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>maphist[j]\ =\ NGRAM\_INVALID\_WID;}
\DoxyCodeLine{00770\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00771\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ set-\/>maphist[j]\ =\ set-\/>widmap[history[j]][set-\/>cur];}
\DoxyCodeLine{00772\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00773\ \ \ \ \ \ \ \ \ score\ =\ ngram\_ng\_prob(set-\/>lms[set-\/>cur],}
\DoxyCodeLine{00774\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ mapwid,\ set-\/>maphist,\ n\_hist,\ n\_used);}
\DoxyCodeLine{00775\ \ \ \ \ \}}
\DoxyCodeLine{00776\ }
\DoxyCodeLine{00777\ \ \ \ \ \textcolor{keywordflow}{return}\ score;}
\DoxyCodeLine{00778\ \}}
\DoxyCodeLine{00779\ }
\DoxyCodeLine{00780\ \textcolor{keyword}{static}\ int32}
\DoxyCodeLine{00781\ ngram\_model\_set\_add\_ug(ngram\_model\_t\ *base,}
\DoxyCodeLine{00782\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ wid,\ int32\ lweight)}
\DoxyCodeLine{00783\ \{}
\DoxyCodeLine{00784\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00785\ \ \ \ \ int32\ *newwid;}
\DoxyCodeLine{00786\ \ \ \ \ int32\ i,\ prob;}
\DoxyCodeLine{00787\ }
\DoxyCodeLine{00788\ \ \ \ \ \textcolor{comment}{/*\ At\ this\ point\ the\ word\ has\ already\ been\ added\ to\ the\ master}}
\DoxyCodeLine{00789\ \textcolor{comment}{\ \ \ \ \ \ \ model\ and\ we\ have\ a\ new\ word\ ID\ for\ it.\ \ Add\ it\ to\ active}}
\DoxyCodeLine{00790\ \textcolor{comment}{\ \ \ \ \ \ \ submodels\ and\ track\ the\ word\ IDs.\ */}}
\DoxyCodeLine{00791\ \ \ \ \ newwid\ =\ ckd\_calloc(set-\/>n\_models,\ \textcolor{keyword}{sizeof}(*newwid));}
\DoxyCodeLine{00792\ \ \ \ \ prob\ =\ base-\/>log\_zero;}
\DoxyCodeLine{00793\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ set-\/>n\_models;\ ++i)\ \{}
\DoxyCodeLine{00794\ \ \ \ \ \ \ \ \ int32\ wprob,\ n\_hist;}
\DoxyCodeLine{00795\ }
\DoxyCodeLine{00796\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Only\ add\ to\ active\ models.\ */}}
\DoxyCodeLine{00797\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (set-\/>cur\ ==\ -\/1\ ||\ set-\/>cur\ ==\ i)\ \{}
\DoxyCodeLine{00798\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Did\ this\ word\ already\ exist?\ */}}
\DoxyCodeLine{00799\ \ \ \ \ \ \ \ \ \ \ \ \ newwid[i]\ =\ ngram\_wid(set-\/>lms[i],\ base-\/>word\_str[wid]);}
\DoxyCodeLine{00800\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (newwid[i]\ ==\ NGRAM\_INVALID\_WID)\ \{}
\DoxyCodeLine{00801\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Add\ it\ to\ the\ submodel.\ */}}
\DoxyCodeLine{00802\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ newwid[i]\ =\ ngram\_model\_add\_word(set-\/>lms[i],\ base-\/>word\_str[wid],}
\DoxyCodeLine{00803\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ logmath\_exp(base-\/>lmath,\ lweight));}
\DoxyCodeLine{00804\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (newwid[i]\ ==\ NGRAM\_INVALID\_WID)\ \{}
\DoxyCodeLine{00805\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_free(newwid);}
\DoxyCodeLine{00806\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ base-\/>log\_zero;}
\DoxyCodeLine{00807\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00808\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00809\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Now\ get\ the\ unigram\ probability\ for\ the\ new\ word\ and\ either}}
\DoxyCodeLine{00810\ \textcolor{comment}{\ \ \ \ \ \ \ \ \ \ \ \ \ *\ interpolate\ it\ or\ use\ it\ (if\ this\ is\ the\ current\ model).\ */}}
\DoxyCodeLine{00811\ \ \ \ \ \ \ \ \ \ \ \ \ wprob\ =\ ngram\_ng\_prob(set-\/>lms[i],\ newwid[i],\ NULL,\ 0,\ \&n\_hist);}
\DoxyCodeLine{00812\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (set-\/>cur\ ==\ i)}
\DoxyCodeLine{00813\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ prob\ =\ wprob;}
\DoxyCodeLine{00814\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \textcolor{keywordflow}{if}\ (set-\/>cur\ ==\ -\/1)}
\DoxyCodeLine{00815\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ prob\ =\ logmath\_add(base-\/>lmath,\ prob,\ set-\/>lweights[i]\ +\ wprob);}
\DoxyCodeLine{00816\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00817\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00818\ \ \ \ \ \ \ \ \ \ \ \ \ newwid[i]\ =\ NGRAM\_INVALID\_WID;}
\DoxyCodeLine{00819\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00820\ \ \ \ \ \}}
\DoxyCodeLine{00821\ \ \ \ \ \textcolor{comment}{/*\ Okay\ we\ have\ the\ word\ IDs\ for\ this\ in\ all\ the\ submodels.\ \ Now}}
\DoxyCodeLine{00822\ \textcolor{comment}{\ \ \ \ \ \ \ do\ some\ complicated\ memory\ mangling\ to\ add\ this\ to\ the}}
\DoxyCodeLine{00823\ \textcolor{comment}{\ \ \ \ \ \ \ widmap.\ */}}
\DoxyCodeLine{00824\ \ \ \ \ set-\/>widmap\ =\ ckd\_realloc(set-\/>widmap,\ base-\/>n\_words\ *\ \textcolor{keyword}{sizeof}(*set-\/>widmap));}
\DoxyCodeLine{00825\ \ \ \ \ set-\/>widmap[0]\ =\ ckd\_realloc(set-\/>widmap[0],}
\DoxyCodeLine{00826\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ base-\/>n\_words}
\DoxyCodeLine{00827\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ *\ set-\/>n\_models}
\DoxyCodeLine{00828\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ *\ \textcolor{keyword}{sizeof}(**set-\/>widmap));}
\DoxyCodeLine{00829\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ base-\/>n\_words;\ ++i)}
\DoxyCodeLine{00830\ \ \ \ \ \ \ \ \ set-\/>widmap[i]\ =\ set-\/>widmap[0]\ +\ i\ *\ set-\/>n\_models;}
\DoxyCodeLine{00831\ \ \ \ \ memcpy(set-\/>widmap[wid],\ newwid,\ set-\/>n\_models\ *\ \textcolor{keyword}{sizeof}(*newwid));}
\DoxyCodeLine{00832\ \ \ \ \ ckd\_free(newwid);}
\DoxyCodeLine{00833\ \ \ \ \ \textcolor{keywordflow}{return}\ prob;}
\DoxyCodeLine{00834\ \}}
\DoxyCodeLine{00835\ }
\DoxyCodeLine{00836\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00837\ ngram\_model\_set\_free(ngram\_model\_t\ *base)}
\DoxyCodeLine{00838\ \{}
\DoxyCodeLine{00839\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00840\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00841\ }
\DoxyCodeLine{00842\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ set-\/>n\_models;\ ++i)}
\DoxyCodeLine{00843\ \ \ \ \ \ \ \ \ ngram\_model\_free(set-\/>lms[i]);}
\DoxyCodeLine{00844\ \ \ \ \ ckd\_free(set-\/>lms);}
\DoxyCodeLine{00845\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ set-\/>n\_models;\ ++i)}
\DoxyCodeLine{00846\ \ \ \ \ \ \ \ \ ckd\_free(set-\/>names[i]);}
\DoxyCodeLine{00847\ \ \ \ \ ckd\_free(set-\/>names);}
\DoxyCodeLine{00848\ \ \ \ \ ckd\_free(set-\/>lweights);}
\DoxyCodeLine{00849\ \ \ \ \ ckd\_free(set-\/>maphist);}
\DoxyCodeLine{00850\ \ \ \ \ ckd\_free\_2d((\textcolor{keywordtype}{void}\ **)set-\/>widmap);}
\DoxyCodeLine{00851\ \}}
\DoxyCodeLine{00852\ }
\DoxyCodeLine{00853\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00854\ ngram\_model\_set\_flush(ngram\_model\_t\ *base)}
\DoxyCodeLine{00855\ \{}
\DoxyCodeLine{00856\ \ \ \ \ ngram\_model\_set\_t\ *set\ =\ (ngram\_model\_set\_t\ *)base;}
\DoxyCodeLine{00857\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00858\ }
\DoxyCodeLine{00859\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ set-\/>n\_models;\ ++i)}
\DoxyCodeLine{00860\ \ \ \ \ \ \ \ \ ngram\_model\_flush(set-\/>lms[i]);}
\DoxyCodeLine{00861\ \}}
\DoxyCodeLine{00862\ }
\DoxyCodeLine{00863\ \textcolor{keyword}{static}\ ngram\_funcs\_t\ ngram\_model\_set\_funcs\ =\ \{}
\DoxyCodeLine{00864\ \ \ \ \ ngram\_model\_set\_free,\ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ free\ */}}
\DoxyCodeLine{00865\ \ \ \ \ ngram\_model\_set\_apply\_weights,\ \textcolor{comment}{/*\ apply\_weights\ */}}
\DoxyCodeLine{00866\ \ \ \ \ ngram\_model\_set\_score,\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ score\ */}}
\DoxyCodeLine{00867\ \ \ \ \ ngram\_model\_set\_raw\_score,\ \ \ \ \ \textcolor{comment}{/*\ raw\_score\ */}}
\DoxyCodeLine{00868\ \ \ \ \ ngram\_model\_set\_add\_ug,\ \ \ \ \ \ \ \ \textcolor{comment}{/*\ add\_ug\ */}}
\DoxyCodeLine{00869\ \ \ \ \ ngram\_model\_set\_flush\ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ flush\ */}}
\DoxyCodeLine{00870\ \};}

\end{DoxyCode}
