\doxysection{ngram\+\_\+model\+\_\+dmp.\+c}
\label{ngram__model__dmp_8c_source}\index{src/libsphinxbase/lm/ngram\_model\_dmp.c@{src/libsphinxbase/lm/ngram\_model\_dmp.c}}

\begin{DoxyCode}{0}
\DoxyCodeLine{00001\ \textcolor{comment}{/*\ -\/*-\/\ c-\/basic-\/offset:\ 4;\ indent-\/tabs-\/mode:\ nil\ -\/*-\/\ */}}
\DoxyCodeLine{00002\ \textcolor{comment}{/*\ ====================================================================}}
\DoxyCodeLine{00003\ \textcolor{comment}{\ *\ Copyright\ (c)\ 1999-\/2007\ Carnegie\ Mellon\ University.\ \ All\ rights}}
\DoxyCodeLine{00004\ \textcolor{comment}{\ *\ reserved.}}
\DoxyCodeLine{00005\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00006\ \textcolor{comment}{\ *\ Redistribution\ and\ use\ in\ source\ and\ binary\ forms,\ with\ or\ without}}
\DoxyCodeLine{00007\ \textcolor{comment}{\ *\ modification,\ are\ permitted\ provided\ that\ the\ following\ conditions}}
\DoxyCodeLine{00008\ \textcolor{comment}{\ *\ are\ met:}}
\DoxyCodeLine{00009\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00010\ \textcolor{comment}{\ *\ 1.\ Redistributions\ of\ source\ code\ must\ retain\ the\ above\ copyright}}
\DoxyCodeLine{00011\ \textcolor{comment}{\ *\ \ \ \ notice,\ this\ list\ of\ conditions\ and\ the\ following\ disclaimer.\ }}
\DoxyCodeLine{00012\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00013\ \textcolor{comment}{\ *\ 2.\ Redistributions\ in\ binary\ form\ must\ reproduce\ the\ above\ copyright}}
\DoxyCodeLine{00014\ \textcolor{comment}{\ *\ \ \ \ notice,\ this\ list\ of\ conditions\ and\ the\ following\ disclaimer\ in}}
\DoxyCodeLine{00015\ \textcolor{comment}{\ *\ \ \ \ the\ documentation\ and/or\ other\ materials\ provided\ with\ the}}
\DoxyCodeLine{00016\ \textcolor{comment}{\ *\ \ \ \ distribution.}}
\DoxyCodeLine{00017\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00018\ \textcolor{comment}{\ *\ This\ work\ was\ supported\ in\ part\ by\ funding\ from\ the\ Defense\ Advanced\ }}
\DoxyCodeLine{00019\ \textcolor{comment}{\ *\ Research\ Projects\ Agency\ and\ the\ National\ Science\ Foundation\ of\ the\ }}
\DoxyCodeLine{00020\ \textcolor{comment}{\ *\ United\ States\ of\ America,\ and\ the\ CMU\ Sphinx\ Speech\ Consortium.}}
\DoxyCodeLine{00021\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00022\ \textcolor{comment}{\ *\ THIS\ SOFTWARE\ IS\ PROVIDED\ BY\ CARNEGIE\ MELLON\ UNIVERSITY\ \`{}\`{}AS\ IS''\ AND\ }}
\DoxyCodeLine{00023\ \textcolor{comment}{\ *\ ANY\ EXPRESSED\ OR\ IMPLIED\ WARRANTIES,\ INCLUDING,\ BUT\ NOT\ LIMITED\ TO,\ }}
\DoxyCodeLine{00024\ \textcolor{comment}{\ *\ THE\ IMPLIED\ WARRANTIES\ OF\ MERCHANTABILITY\ AND\ FITNESS\ FOR\ A\ PARTICULAR}}
\DoxyCodeLine{00025\ \textcolor{comment}{\ *\ PURPOSE\ ARE\ DISCLAIMED.\ \ IN\ NO\ EVENT\ SHALL\ CARNEGIE\ MELLON\ UNIVERSITY}}
\DoxyCodeLine{00026\ \textcolor{comment}{\ *\ NOR\ ITS\ EMPLOYEES\ BE\ LIABLE\ FOR\ ANY\ DIRECT,\ INDIRECT,\ INCIDENTAL,}}
\DoxyCodeLine{00027\ \textcolor{comment}{\ *\ SPECIAL,\ EXEMPLARY,\ OR\ CONSEQUENTIAL\ DAMAGES\ (INCLUDING,\ BUT\ NOT\ }}
\DoxyCodeLine{00028\ \textcolor{comment}{\ *\ LIMITED\ TO,\ PROCUREMENT\ OF\ SUBSTITUTE\ GOODS\ OR\ SERVICES;\ LOSS\ OF\ USE,\ }}
\DoxyCodeLine{00029\ \textcolor{comment}{\ *\ DATA,\ OR\ PROFITS;\ OR\ BUSINESS\ INTERRUPTION)\ HOWEVER\ CAUSED\ AND\ ON\ ANY\ }}
\DoxyCodeLine{00030\ \textcolor{comment}{\ *\ THEORY\ OF\ LIABILITY,\ WHETHER\ IN\ CONTRACT,\ STRICT\ LIABILITY,\ OR\ TORT\ }}
\DoxyCodeLine{00031\ \textcolor{comment}{\ *\ (INCLUDING\ NEGLIGENCE\ OR\ OTHERWISE)\ ARISING\ IN\ ANY\ WAY\ OUT\ OF\ THE\ USE\ }}
\DoxyCodeLine{00032\ \textcolor{comment}{\ *\ OF\ THIS\ SOFTWARE,\ EVEN\ IF\ ADVISED\ OF\ THE\ POSSIBILITY\ OF\ SUCH\ DAMAGE.}}
\DoxyCodeLine{00033\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00034\ \textcolor{comment}{\ *\ ====================================================================}}
\DoxyCodeLine{00035\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00036\ \textcolor{comment}{\ */}}
\DoxyCodeLine{00037\ \textcolor{comment}{/*}}
\DoxyCodeLine{00038\ \textcolor{comment}{\ *\ \(\backslash\)file\ ngram\_model\_dmp.c\ DMP\ format\ language\ models}}
\DoxyCodeLine{00039\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00040\ \textcolor{comment}{\ *\ Author:\ David\ Huggins-\/Daines\ <dhuggins@cs.cmu.edu>}}
\DoxyCodeLine{00041\ \textcolor{comment}{\ */}}
\DoxyCodeLine{00042\ }
\DoxyCodeLine{00043\ \textcolor{preprocessor}{\#include\ <assert.h>}}
\DoxyCodeLine{00044\ \textcolor{preprocessor}{\#include\ <stdio.h>}}
\DoxyCodeLine{00045\ \textcolor{preprocessor}{\#include\ <string.h>}}
\DoxyCodeLine{00046\ \textcolor{preprocessor}{\#include\ <stdlib.h>}}
\DoxyCodeLine{00047\ \textcolor{preprocessor}{\#include\ <limits.h>}}
\DoxyCodeLine{00048\ }
\DoxyCodeLine{00049\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/ckd\_alloc.h"{}}}
\DoxyCodeLine{00050\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/pio.h"{}}}
\DoxyCodeLine{00051\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/err.h"{}}}
\DoxyCodeLine{00052\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/byteorder.h"{}}}
\DoxyCodeLine{00053\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/listelem\_alloc.h"{}}}
\DoxyCodeLine{00054\ }
\DoxyCodeLine{00055\ \textcolor{preprocessor}{\#include\ "{}ngram\_model\_dmp.h"{}}}
\DoxyCodeLine{00056\ }
\DoxyCodeLine{00057\ \textcolor{keyword}{static}\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ darpa\_hdr[]\ =\ \textcolor{stringliteral}{"{}Darpa\ Trigram\ LM"{}};}
\DoxyCodeLine{00058\ \textcolor{keyword}{static}\ ngram\_funcs\_t\ ngram\_model\_dmp\_funcs;}
\DoxyCodeLine{00059\ }
\DoxyCodeLine{00060\ \textcolor{preprocessor}{\#define\ TSEG\_BASE(m,b)\ \ \ \ \ \ \ \ \ \ ((m)-\/>lm3g.tseg\_base[(b)>>LOG\_BG\_SEG\_SZ])}}
\DoxyCodeLine{00061\ \textcolor{preprocessor}{\#define\ FIRST\_BG(m,u)\ \ \ \ \ \ \ \ \ \ \ ((m)-\/>lm3g.unigrams[u].bigrams)}}
\DoxyCodeLine{00062\ \textcolor{preprocessor}{\#define\ FIRST\_TG(m,b)\ \ \ \ \ \ \ \ \ \ \ (TSEG\_BASE((m),(b))+((m)-\/>lm3g.bigrams[b].trigrams))}}
\DoxyCodeLine{00063\ }
\DoxyCodeLine{00064\ \textcolor{keyword}{static}\ unigram\_t\ *}
\DoxyCodeLine{00065\ new\_unigram\_table(int32\ n\_ug)}
\DoxyCodeLine{00066\ \{}
\DoxyCodeLine{00067\ \ \ \ \ unigram\_t\ *table;}
\DoxyCodeLine{00068\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00069\ }
\DoxyCodeLine{00070\ \ \ \ \ table\ =\ ckd\_calloc(n\_ug,\ \textcolor{keyword}{sizeof}(unigram\_t));}
\DoxyCodeLine{00071\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ n\_ug;\ i++)\ \{}
\DoxyCodeLine{00072\ \ \ \ \ \ \ \ \ table[i].prob1.f\ =\ -\/99.0;}
\DoxyCodeLine{00073\ \ \ \ \ \ \ \ \ table[i].bo\_wt1.f\ =\ -\/99.0;}
\DoxyCodeLine{00074\ \ \ \ \ \}}
\DoxyCodeLine{00075\ \ \ \ \ \textcolor{keywordflow}{return}\ table;}
\DoxyCodeLine{00076\ \}}
\DoxyCodeLine{00077\ }
\DoxyCodeLine{00078\ ngram\_model\_t\ *}
\DoxyCodeLine{00079\ ngram\_model\_dmp\_read(cmd\_ln\_t\ *config,}
\DoxyCodeLine{00080\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *file\_name,}
\DoxyCodeLine{00081\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ logmath\_t\ *lmath)}
\DoxyCodeLine{00082\ \{}
\DoxyCodeLine{00083\ \ \ \ \ ngram\_model\_t\ *base;}
\DoxyCodeLine{00084\ \ \ \ \ ngram\_model\_dmp\_t\ *model;}
\DoxyCodeLine{00085\ \ \ \ \ FILE\ *fp;}
\DoxyCodeLine{00086\ \ \ \ \ \textcolor{keywordtype}{int}\ do\_mmap,\ do\_swap;}
\DoxyCodeLine{00087\ \ \ \ \ int32\ is\_pipe;}
\DoxyCodeLine{00088\ \ \ \ \ int32\ i,\ j,\ k,\ vn,\ n,\ ts;}
\DoxyCodeLine{00089\ \ \ \ \ int32\ n\_unigram;}
\DoxyCodeLine{00090\ \ \ \ \ int32\ n\_bigram;}
\DoxyCodeLine{00091\ \ \ \ \ int32\ n\_trigram;}
\DoxyCodeLine{00092\ \ \ \ \ \textcolor{keywordtype}{char}\ str[1024];}
\DoxyCodeLine{00093\ \ \ \ \ unigram\_t\ *ugptr;}
\DoxyCodeLine{00094\ \ \ \ \ bigram\_t\ *bgptr;}
\DoxyCodeLine{00095\ \ \ \ \ trigram\_t\ *tgptr;}
\DoxyCodeLine{00096\ \ \ \ \ \textcolor{keywordtype}{char}\ *tmp\_word\_str;}
\DoxyCodeLine{00097\ \ \ \ \ \textcolor{keywordtype}{char}\ *map\_base\ =\ NULL;}
\DoxyCodeLine{00098\ \ \ \ \ \textcolor{keywordtype}{size\_t}\ offset\ =\ 0;}
\DoxyCodeLine{00099\ }
\DoxyCodeLine{00100\ \ \ \ \ base\ =\ NULL;}
\DoxyCodeLine{00101\ \ \ \ \ do\_mmap\ =\ FALSE;}
\DoxyCodeLine{00102\ \ \ \ \ \textcolor{keywordflow}{if}\ (config)}
\DoxyCodeLine{00103\ \ \ \ \ \ \ \ \ do\_mmap\ =\ cmd\_ln\_boolean\_r(config,\ \textcolor{stringliteral}{"{}-\/mmap"{}});}
\DoxyCodeLine{00104\ }
\DoxyCodeLine{00105\ \ \ \ \ \textcolor{keywordflow}{if}\ ((fp\ =\ fopen\_comp(file\_name,\ \textcolor{stringliteral}{"{}rb"{}},\ \&is\_pipe))\ ==\ NULL)\ \{}
\DoxyCodeLine{00106\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Dump\ file\ \%s\ not\ found\(\backslash\)n"{}},\ file\_name);}
\DoxyCodeLine{00107\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00108\ \ \ \ \ \}}
\DoxyCodeLine{00109\ }
\DoxyCodeLine{00110\ \ \ \ \ \textcolor{keywordflow}{if}\ (is\_pipe\ \&\&\ do\_mmap)\ \{}
\DoxyCodeLine{00111\ \ \ \ \ \ \ \ \ E\_WARN(\textcolor{stringliteral}{"{}Dump\ file\ is\ compressed,\ will\ not\ use\ memory-\/mapped\ I/O\(\backslash\)n"{}});}
\DoxyCodeLine{00112\ \ \ \ \ \ \ \ \ do\_mmap\ =\ 0;}
\DoxyCodeLine{00113\ \ \ \ \ \}}
\DoxyCodeLine{00114\ }
\DoxyCodeLine{00115\ \ \ \ \ do\_swap\ =\ FALSE;}
\DoxyCodeLine{00116\ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(\&k,\ \textcolor{keyword}{sizeof}(k),\ 1,\ fp)\ !=\ 1)}
\DoxyCodeLine{00117\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00118\ \ \ \ \ \textcolor{keywordflow}{if}\ (k\ !=\ strlen(darpa\_hdr)+1)\ \{}
\DoxyCodeLine{00119\ \ \ \ \ \ \ \ \ SWAP\_INT32(\&k);}
\DoxyCodeLine{00120\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (k\ !=\ strlen(darpa\_hdr)+1)\ \{}
\DoxyCodeLine{00121\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Wrong\ magic\ header\ size\ number\ \%x:\ \%s\ is\ not\ a\ dump\ file\(\backslash\)n"{}},\ k,\ file\_name);}
\DoxyCodeLine{00122\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00123\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00124\ \ \ \ \ \ \ \ \ do\_swap\ =\ 1;}
\DoxyCodeLine{00125\ \ \ \ \ \}}
\DoxyCodeLine{00126\ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(str,\ 1,\ k,\ fp)\ !=\ (\textcolor{keywordtype}{size\_t})\ k)\ \{}
\DoxyCodeLine{00127\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Cannot\ read\ header\(\backslash\)n"{}});}
\DoxyCodeLine{00128\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00129\ \ \ \ \ \}}
\DoxyCodeLine{00130\ \ \ \ \ \textcolor{keywordflow}{if}\ (strncmp(str,\ darpa\_hdr,\ k)\ !=\ 0)\ \{}
\DoxyCodeLine{00131\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Wrong\ header\ \%s:\ \%s\ is\ not\ a\ dump\ file\(\backslash\)n"{}},\ darpa\_hdr);}
\DoxyCodeLine{00132\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00133\ \ \ \ \ \}}
\DoxyCodeLine{00134\ }
\DoxyCodeLine{00135\ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_mmap)\ \{}
\DoxyCodeLine{00136\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)\ \{}
\DoxyCodeLine{00137\ \ \ \ \ \ \ \ \ \ \ \ \ E\_INFO}
\DoxyCodeLine{00138\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (\textcolor{stringliteral}{"{}Byteswapping\ required,\ will\ not\ use\ memory-\/mapped\ I/O\ for\ LM\ file\(\backslash\)n"{}});}
\DoxyCodeLine{00139\ \ \ \ \ \ \ \ \ \ \ \ \ do\_mmap\ =\ 0;}
\DoxyCodeLine{00140\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00141\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00142\ \ \ \ \ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}Will\ use\ memory-\/mapped\ I/O\ for\ LM\ file\(\backslash\)n"{}});}
\DoxyCodeLine{00143\ \textcolor{preprocessor}{\#ifdef\ \_\_ADSPBLACKFIN\_\_\ }\textcolor{comment}{/*\ This\ is\ true\ for\ both\ VisualDSP++\ and\ uClinux.\ */}\textcolor{preprocessor}{}}
\DoxyCodeLine{00144\ \ \ \ \ \ \ \ \ \ \ \ \ E\_FATAL(\textcolor{stringliteral}{"{}memory\ mapping\ is\ not\ supported\ at\ the\ moment."{}});}
\DoxyCodeLine{00145\ \textcolor{preprocessor}{\#else}}
\DoxyCodeLine{00146\ \textcolor{preprocessor}{\#endif}}
\DoxyCodeLine{00147\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00148\ \ \ \ \ \}}
\DoxyCodeLine{00149\ }
\DoxyCodeLine{00150\ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(\&k,\ \textcolor{keyword}{sizeof}(k),\ 1,\ fp)\ !=\ 1)}
\DoxyCodeLine{00151\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00152\ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)\ SWAP\_INT32(\&k);}
\DoxyCodeLine{00153\ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(str,\ 1,\ k,\ fp)\ !=\ (\textcolor{keywordtype}{size\_t})\ k)\ \{}
\DoxyCodeLine{00154\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Cannot\ read\ LM\ filename\ in\ header\(\backslash\)n"{}});}
\DoxyCodeLine{00155\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00156\ \ \ \ \ \}}
\DoxyCodeLine{00157\ }
\DoxyCodeLine{00158\ \ \ \ \ \textcolor{comment}{/*\ read\ version\#,\ if\ present\ (must\ be\ <=\ 0)\ */}}
\DoxyCodeLine{00159\ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(\&vn,\ \textcolor{keyword}{sizeof}(vn),\ 1,\ fp)\ !=\ 1)}
\DoxyCodeLine{00160\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00161\ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)\ SWAP\_INT32(\&vn);}
\DoxyCodeLine{00162\ \ \ \ \ \textcolor{keywordflow}{if}\ (vn\ <=\ 0)\ \{}
\DoxyCodeLine{00163\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ read\ and\ don't\ compare\ timestamps\ (we\ don't\ care)\ */}}
\DoxyCodeLine{00164\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(\&ts,\ \textcolor{keyword}{sizeof}(ts),\ 1,\ fp)\ !=\ 1)}
\DoxyCodeLine{00165\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00166\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)\ SWAP\_INT32(\&ts);}
\DoxyCodeLine{00167\ }
\DoxyCodeLine{00168\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ read\ and\ skip\ format\ description\ */}}
\DoxyCodeLine{00169\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (;;)\ \{}
\DoxyCodeLine{00170\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(\&k,\ \textcolor{keyword}{sizeof}(k),\ 1,\ fp)\ !=\ 1)}
\DoxyCodeLine{00171\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00172\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)\ SWAP\_INT32(\&k);}
\DoxyCodeLine{00173\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (k\ ==\ 0)}
\DoxyCodeLine{00174\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00175\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(str,\ 1,\ k,\ fp)\ !=\ (\textcolor{keywordtype}{size\_t})\ k)\ \{}
\DoxyCodeLine{00176\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Failed\ to\ read\ word\(\backslash\)n"{}});}
\DoxyCodeLine{00177\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00178\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00179\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00180\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ read\ model-\/>ucount\ */}}
\DoxyCodeLine{00181\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(\&n\_unigram,\ \textcolor{keyword}{sizeof}(n\_unigram),\ 1,\ fp)\ !=\ 1)}
\DoxyCodeLine{00182\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00183\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)\ SWAP\_INT32(\&n\_unigram);}
\DoxyCodeLine{00184\ \ \ \ \ \}}
\DoxyCodeLine{00185\ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00186\ \ \ \ \ \ \ \ \ n\_unigram\ =\ vn;}
\DoxyCodeLine{00187\ \ \ \ \ \}}
\DoxyCodeLine{00188\ }
\DoxyCodeLine{00189\ \ \ \ \ \textcolor{comment}{/*\ read\ model-\/>bcount,\ tcount\ */}}
\DoxyCodeLine{00190\ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(\&n\_bigram,\ \textcolor{keyword}{sizeof}(n\_bigram),\ 1,\ fp)\ !=\ 1)}
\DoxyCodeLine{00191\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00192\ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)\ SWAP\_INT32(\&n\_bigram);}
\DoxyCodeLine{00193\ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(\&n\_trigram,\ \textcolor{keyword}{sizeof}(n\_trigram),\ 1,\ fp)\ !=\ 1)}
\DoxyCodeLine{00194\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00195\ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)\ SWAP\_INT32(\&n\_trigram);}
\DoxyCodeLine{00196\ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}ngrams\ 1=\%d,\ 2=\%d,\ 3=\%d\(\backslash\)n"{}},\ n\_unigram,\ n\_bigram,\ n\_trigram);}
\DoxyCodeLine{00197\ }
\DoxyCodeLine{00198\ \ \ \ \ \textcolor{comment}{/*\ Allocate\ space\ for\ LM,\ including\ initial\ OOVs\ and\ placeholders;\ initialize\ it\ */}}
\DoxyCodeLine{00199\ \ \ \ \ model\ =\ ckd\_calloc(1,\ \textcolor{keyword}{sizeof}(*model));}
\DoxyCodeLine{00200\ \ \ \ \ base\ =\ \&model-\/>base;}
\DoxyCodeLine{00201\ \ \ \ \ \textcolor{keywordflow}{if}\ (n\_trigram\ >\ 0)}
\DoxyCodeLine{00202\ \ \ \ \ \ \ \ \ n\ =\ 3;}
\DoxyCodeLine{00203\ \ \ \ \ \textcolor{keywordflow}{else}\ \textcolor{keywordflow}{if}\ (n\_bigram\ >\ 0)}
\DoxyCodeLine{00204\ \ \ \ \ \ \ \ \ n\ =\ 2;}
\DoxyCodeLine{00205\ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00206\ \ \ \ \ \ \ \ \ n\ =\ 1;}
\DoxyCodeLine{00207\ \ \ \ \ ngram\_model\_init(base,\ \&ngram\_model\_dmp\_funcs,\ lmath,\ n,\ n\_unigram);}
\DoxyCodeLine{00208\ \ \ \ \ base-\/>n\_counts[0]\ =\ n\_unigram;}
\DoxyCodeLine{00209\ \ \ \ \ base-\/>n\_counts[1]\ =\ n\_bigram;}
\DoxyCodeLine{00210\ \ \ \ \ base-\/>n\_counts[2]\ =\ n\_trigram;}
\DoxyCodeLine{00211\ }
\DoxyCodeLine{00212\ \ \ \ \ \textcolor{comment}{/*\ read\ unigrams\ (always\ in\ memory,\ as\ they\ contain\ dictionary}}
\DoxyCodeLine{00213\ \textcolor{comment}{\ \ \ \ \ *\ mappings\ that\ can't\ be\ precomputed,\ and\ also\ could\ have\ OOVs\ added)\ */}}
\DoxyCodeLine{00214\ \ \ \ \ model-\/>lm3g.unigrams\ =\ new\_unigram\_table(n\_unigram\ +\ 1);}
\DoxyCodeLine{00215\ \ \ \ \ ugptr\ =\ model-\/>lm3g.unigrams;}
\DoxyCodeLine{00216\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <=\ n\_unigram;\ ++i)\ \{}
\DoxyCodeLine{00217\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Skip\ over\ the\ mapping\ ID,\ we\ don't\ care\ about\ it.\ */}}
\DoxyCodeLine{00218\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(ugptr,\ \textcolor{keyword}{sizeof}(int32),\ 1,\ fp)\ !=\ 1)\ \{}
\DoxyCodeLine{00219\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Failed\ to\ read\ maping\ id\ \%d\(\backslash\)n"{}},\ i);}
\DoxyCodeLine{00220\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00221\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00222\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Read\ the\ actual\ unigram\ structure.\ */}}
\DoxyCodeLine{00223\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(ugptr,\ \textcolor{keyword}{sizeof}(unigram\_t),\ 1,\ fp)\ !=\ 1)\ \ \{}
\DoxyCodeLine{00224\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Failed\ to\ read\ unigrams\ data\(\backslash\)n"{}});}
\DoxyCodeLine{00225\ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_model\_free(base);}
\DoxyCodeLine{00226\ \ \ \ \ \ \ \ \ \ \ \ \ fclose\_comp(fp,\ is\_pipe);}
\DoxyCodeLine{00227\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00228\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00229\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Byte\ swap\ if\ necessary.\ */}}
\DoxyCodeLine{00230\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)\ \{}
\DoxyCodeLine{00231\ \ \ \ \ \ \ \ \ \ \ \ \ SWAP\_INT32(\&ugptr-\/>prob1.l);}
\DoxyCodeLine{00232\ \ \ \ \ \ \ \ \ \ \ \ \ SWAP\_INT32(\&ugptr-\/>bo\_wt1.l);}
\DoxyCodeLine{00233\ \ \ \ \ \ \ \ \ \ \ \ \ SWAP\_INT32(\&ugptr-\/>bigrams);}
\DoxyCodeLine{00234\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00235\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Convert\ values\ to\ log.\ */}}
\DoxyCodeLine{00236\ \ \ \ \ \ \ \ \ ugptr-\/>prob1.l\ =\ logmath\_log10\_to\_log(lmath,\ ugptr-\/>prob1.f);}
\DoxyCodeLine{00237\ \ \ \ \ \ \ \ \ ugptr-\/>bo\_wt1.l\ =\ logmath\_log10\_to\_log(lmath,\ ugptr-\/>bo\_wt1.f);}
\DoxyCodeLine{00238\ \ \ \ \ \ \ \ \ E\_DEBUG(2,\ (\textcolor{stringliteral}{"{}ug\ \%d:\ prob\ \%d\ bo\ \%d\ bigrams\ \%d\(\backslash\)n"{}},}
\DoxyCodeLine{00239\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ i,\ ugptr-\/>prob1.l,\ ugptr-\/>bo\_wt1.l,\ ugptr-\/>bigrams));}
\DoxyCodeLine{00240\ \ \ \ \ \ \ \ \ ++ugptr;}
\DoxyCodeLine{00241\ \ \ \ \ \}}
\DoxyCodeLine{00242\ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ LM.unigrams(+trailer)\ read\(\backslash\)n"{}},\ n\_unigram);}
\DoxyCodeLine{00243\ }
\DoxyCodeLine{00244\ \ \ \ \ \textcolor{comment}{/*\ Now\ mmap()\ the\ file\ and\ read\ in\ the\ rest\ of\ the\ (read-\/only)\ stuff.\ */}}
\DoxyCodeLine{00245\ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_mmap)\ \{}
\DoxyCodeLine{00246\ \ \ \ \ \ \ \ \ offset\ =\ ftell(fp);}
\DoxyCodeLine{00247\ }
\DoxyCodeLine{00248\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Check\ for\ improper\ word\ alignment.\ */}}
\DoxyCodeLine{00249\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (offset\ \&\ 0x3)\ \{}
\DoxyCodeLine{00250\ \ \ \ \ \ \ \ \ \ \ \ \ E\_WARN(\textcolor{stringliteral}{"{}-\/mmap\ specified,\ but\ trigram\ index\ is\ not\ word-\/aligned.\ \ Will\ not\ memory-\/map.\(\backslash\)n"{}});}
\DoxyCodeLine{00251\ \ \ \ \ \ \ \ \ \ \ \ \ do\_mmap\ =\ FALSE;}
\DoxyCodeLine{00252\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00253\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00254\ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>dump\_mmap\ =\ mmio\_file\_read(file\_name);}
\DoxyCodeLine{00255\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (model-\/>dump\_mmap\ ==\ NULL)\ \{}
\DoxyCodeLine{00256\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\_mmap\ =\ FALSE;}
\DoxyCodeLine{00257\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00258\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00259\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ map\_base\ =\ mmio\_file\_ptr(model-\/>dump\_mmap);}
\DoxyCodeLine{00260\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00261\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00262\ \ \ \ \ \}}
\DoxyCodeLine{00263\ \ \ \ \ }
\DoxyCodeLine{00264\ \ \ \ \ \textcolor{keywordflow}{if}\ (n\_bigram\ >\ 0)\ \{}
\DoxyCodeLine{00265\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ read\ bigrams\ */}}
\DoxyCodeLine{00266\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_mmap)\ \{}
\DoxyCodeLine{00267\ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>lm3g.bigrams\ =\ (bigram\_t\ *)\ (map\_base\ +\ offset);}
\DoxyCodeLine{00268\ \ \ \ \ \ \ \ \ \ \ \ \ offset\ +=\ (n\_bigram\ +\ 1)\ *\ \textcolor{keyword}{sizeof}(bigram\_t);}
\DoxyCodeLine{00269\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00270\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00271\ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>lm3g.bigrams\ =}
\DoxyCodeLine{00272\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_calloc(n\_bigram\ +\ 1,\ \textcolor{keyword}{sizeof}(bigram\_t));}
\DoxyCodeLine{00273\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(model-\/>lm3g.bigrams,\ \textcolor{keyword}{sizeof}(bigram\_t),\ n\_bigram\ +\ 1,\ fp)}
\DoxyCodeLine{00274\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ !=\ (\textcolor{keywordtype}{size\_t})\ n\_bigram\ +\ 1)\ \{}
\DoxyCodeLine{00275\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Failed\ to\ read\ bigrams\ data\(\backslash\)n"{}});}
\DoxyCodeLine{00276\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00277\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00278\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)\ \{}
\DoxyCodeLine{00279\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0,\ bgptr\ =\ model-\/>lm3g.bigrams;\ i\ <=\ n\_bigram;}
\DoxyCodeLine{00280\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ i++,\ bgptr++)\ \{}
\DoxyCodeLine{00281\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ SWAP\_INT16(\&bgptr-\/>wid);}
\DoxyCodeLine{00282\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ SWAP\_INT16(\&bgptr-\/>prob2);}
\DoxyCodeLine{00283\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ SWAP\_INT16(\&bgptr-\/>bo\_wt2);}
\DoxyCodeLine{00284\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ SWAP\_INT16(\&bgptr-\/>trigrams);}
\DoxyCodeLine{00285\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00286\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00287\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00288\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ LM.bigrams(+trailer)\ read\(\backslash\)n"{}},\ n\_bigram);}
\DoxyCodeLine{00289\ \ \ \ \ \}}
\DoxyCodeLine{00290\ }
\DoxyCodeLine{00291\ \ \ \ \ \textcolor{comment}{/*\ read\ trigrams\ */}}
\DoxyCodeLine{00292\ \ \ \ \ \textcolor{keywordflow}{if}\ (n\_trigram\ >\ 0)\ \{}
\DoxyCodeLine{00293\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_mmap)\ \{}
\DoxyCodeLine{00294\ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>lm3g.trigrams\ =\ (trigram\_t\ *)\ (map\_base\ +\ offset);}
\DoxyCodeLine{00295\ \ \ \ \ \ \ \ \ \ \ \ \ offset\ +=\ n\_trigram\ *\ \textcolor{keyword}{sizeof}(trigram\_t);}
\DoxyCodeLine{00296\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00297\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00298\ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>lm3g.trigrams\ =}
\DoxyCodeLine{00299\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_calloc(n\_trigram,\ \textcolor{keyword}{sizeof}(trigram\_t));}
\DoxyCodeLine{00300\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread}
\DoxyCodeLine{00301\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (model-\/>lm3g.trigrams,\ \textcolor{keyword}{sizeof}(trigram\_t),\ n\_trigram,\ fp)}
\DoxyCodeLine{00302\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ !=\ (\textcolor{keywordtype}{size\_t})\ n\_trigram)\ \{}
\DoxyCodeLine{00303\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Failed\ to\ read\ trigrams\ data\(\backslash\)n"{}});}
\DoxyCodeLine{00304\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00305\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00306\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)\ \{}
\DoxyCodeLine{00307\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0,\ tgptr\ =\ model-\/>lm3g.trigrams;\ i\ <\ n\_trigram;}
\DoxyCodeLine{00308\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ i++,\ tgptr++)\ \{}
\DoxyCodeLine{00309\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ SWAP\_INT16(\&tgptr-\/>wid);}
\DoxyCodeLine{00310\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ SWAP\_INT16(\&tgptr-\/>prob3);}
\DoxyCodeLine{00311\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00312\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00313\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00314\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ LM.trigrams\ read\(\backslash\)n"{}},\ n\_trigram);}
\DoxyCodeLine{00315\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Initialize\ tginfo\ */}}
\DoxyCodeLine{00316\ \ \ \ \ \ \ \ \ model-\/>lm3g.tginfo\ =\ ckd\_calloc(n\_unigram,\ \textcolor{keyword}{sizeof}(tginfo\_t\ *));}
\DoxyCodeLine{00317\ \ \ \ \ \ \ \ \ model-\/>lm3g.le\ =\ listelem\_alloc\_init(\textcolor{keyword}{sizeof}(tginfo\_t));}
\DoxyCodeLine{00318\ \ \ \ \ \}}
\DoxyCodeLine{00319\ }
\DoxyCodeLine{00320\ \ \ \ \ \textcolor{keywordflow}{if}\ (n\_bigram\ >\ 0)\ \{}
\DoxyCodeLine{00321\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ read\ n\_prob2\ and\ prob2\ array\ (in\ memory)\ */}}
\DoxyCodeLine{00322\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_mmap)}
\DoxyCodeLine{00323\ \ \ \ \ \ \ \ \ \ \ \ \ fseek(fp,\ offset,\ SEEK\_SET);}
\DoxyCodeLine{00324\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(\&k,\ \textcolor{keyword}{sizeof}(k),\ 1,\ fp)\ !=\ 1)}
\DoxyCodeLine{00325\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00326\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)\ SWAP\_INT32(\&k);}
\DoxyCodeLine{00327\ \ \ \ \ \ \ \ \ model-\/>lm3g.n\_prob2\ =\ k;}
\DoxyCodeLine{00328\ \ \ \ \ \ \ \ \ model-\/>lm3g.prob2\ =\ ckd\_calloc(k,\ \textcolor{keyword}{sizeof}(*model-\/>lm3g.prob2));}
\DoxyCodeLine{00329\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(model-\/>lm3g.prob2,\ \textcolor{keyword}{sizeof}(*model-\/>lm3g.prob2),\ k,\ fp)\ !=\ (\textcolor{keywordtype}{size\_t})\ k)\ \{}
\DoxyCodeLine{00330\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}fread(prob2)\ failed\(\backslash\)n"{}});}
\DoxyCodeLine{00331\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00332\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00333\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ k;\ i++)\ \{}
\DoxyCodeLine{00334\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)}
\DoxyCodeLine{00335\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ SWAP\_INT32(\&model-\/>lm3g.prob2[i].l);}
\DoxyCodeLine{00336\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Convert\ values\ to\ log.\ */}}
\DoxyCodeLine{00337\ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>lm3g.prob2[i].l\ =\ logmath\_log10\_to\_log(lmath,\ model-\/>lm3g.prob2[i].f);}
\DoxyCodeLine{00338\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00339\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ LM.prob2\ entries\ read\(\backslash\)n"{}},\ k);}
\DoxyCodeLine{00340\ \ \ \ \ \}}
\DoxyCodeLine{00341\ }
\DoxyCodeLine{00342\ \ \ \ \ \textcolor{comment}{/*\ read\ n\_bo\_wt2\ and\ bo\_wt2\ array\ (in\ memory)\ */}}
\DoxyCodeLine{00343\ \ \ \ \ \textcolor{keywordflow}{if}\ (base-\/>n\ >\ 2)\ \{}
\DoxyCodeLine{00344\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(\&k,\ \textcolor{keyword}{sizeof}(k),\ 1,\ fp)\ !=\ 1)}
\DoxyCodeLine{00345\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00346\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)\ SWAP\_INT32(\&k);}
\DoxyCodeLine{00347\ \ \ \ \ \ \ \ \ model-\/>lm3g.n\_bo\_wt2\ =\ k;}
\DoxyCodeLine{00348\ \ \ \ \ \ \ \ \ model-\/>lm3g.bo\_wt2\ =\ ckd\_calloc(k,\ \textcolor{keyword}{sizeof}(*model-\/>lm3g.bo\_wt2));}
\DoxyCodeLine{00349\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(model-\/>lm3g.bo\_wt2,\ \textcolor{keyword}{sizeof}(*model-\/>lm3g.bo\_wt2),\ k,\ fp)\ !=\ (\textcolor{keywordtype}{size\_t})\ k)\ \{}
\DoxyCodeLine{00350\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Failed\ to\ read\ backoff\ weights\(\backslash\)n"{}});}
\DoxyCodeLine{00351\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00352\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00353\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ k;\ i++)\ \{}
\DoxyCodeLine{00354\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)}
\DoxyCodeLine{00355\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ SWAP\_INT32(\&model-\/>lm3g.bo\_wt2[i].l);}
\DoxyCodeLine{00356\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Convert\ values\ to\ log.\ */}}
\DoxyCodeLine{00357\ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>lm3g.bo\_wt2[i].l\ =\ logmath\_log10\_to\_log(lmath,\ model-\/>lm3g.bo\_wt2[i].f);}
\DoxyCodeLine{00358\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00359\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ LM.bo\_wt2\ entries\ read\(\backslash\)n"{}},\ k);}
\DoxyCodeLine{00360\ \ \ \ \ \}}
\DoxyCodeLine{00361\ }
\DoxyCodeLine{00362\ \ \ \ \ \textcolor{comment}{/*\ read\ n\_prob3\ and\ prob3\ array\ (in\ memory)\ */}}
\DoxyCodeLine{00363\ \ \ \ \ \textcolor{keywordflow}{if}\ (base-\/>n\ >\ 2)\ \{}
\DoxyCodeLine{00364\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(\&k,\ \textcolor{keyword}{sizeof}(k),\ 1,\ fp)\ !=\ 1)}
\DoxyCodeLine{00365\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00366\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)\ SWAP\_INT32(\&k);}
\DoxyCodeLine{00367\ \ \ \ \ \ \ \ \ model-\/>lm3g.n\_prob3\ =\ k;}
\DoxyCodeLine{00368\ \ \ \ \ \ \ \ \ model-\/>lm3g.prob3\ =\ ckd\_calloc(k,\ \textcolor{keyword}{sizeof}(*model-\/>lm3g.prob3));}
\DoxyCodeLine{00369\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(model-\/>lm3g.prob3,\ \textcolor{keyword}{sizeof}(*model-\/>lm3g.prob3),\ k,\ fp)\ !=\ (\textcolor{keywordtype}{size\_t})\ k)\ \{}
\DoxyCodeLine{00370\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Failed\ to\ read\ trigram\ probability\(\backslash\)n"{}});}
\DoxyCodeLine{00371\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00372\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00373\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ k;\ i++)\ \{}
\DoxyCodeLine{00374\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)}
\DoxyCodeLine{00375\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ SWAP\_INT32(\&model-\/>lm3g.prob3[i].l);}
\DoxyCodeLine{00376\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Convert\ values\ to\ log.\ */}}
\DoxyCodeLine{00377\ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>lm3g.prob3[i].l\ =\ logmath\_log10\_to\_log(lmath,\ model-\/>lm3g.prob3[i].f);}
\DoxyCodeLine{00378\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00379\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ LM.prob3\ entries\ read\(\backslash\)n"{}},\ k);}
\DoxyCodeLine{00380\ \ \ \ \ \}}
\DoxyCodeLine{00381\ }
\DoxyCodeLine{00382\ \ \ \ \ \textcolor{comment}{/*\ read\ tseg\_base\ size\ and\ tseg\_base\ */}}
\DoxyCodeLine{00383\ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_mmap)}
\DoxyCodeLine{00384\ \ \ \ \ \ \ \ \ offset\ =\ ftell(fp);}
\DoxyCodeLine{00385\ \ \ \ \ \textcolor{keywordflow}{if}\ (n\_trigram\ >\ 0)\ \{}
\DoxyCodeLine{00386\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_mmap)\ \{}
\DoxyCodeLine{00387\ \ \ \ \ \ \ \ \ \ \ \ \ memcpy(\&k,\ map\_base\ +\ offset,\ \textcolor{keyword}{sizeof}(k));}
\DoxyCodeLine{00388\ \ \ \ \ \ \ \ \ \ \ \ \ offset\ +=\ \textcolor{keyword}{sizeof}(int32);}
\DoxyCodeLine{00389\ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>lm3g.tseg\_base\ =\ (int32\ *)\ (map\_base\ +\ offset);}
\DoxyCodeLine{00390\ \ \ \ \ \ \ \ \ \ \ \ \ offset\ +=\ k\ *\ \textcolor{keyword}{sizeof}(int32);}
\DoxyCodeLine{00391\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00392\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00393\ \ \ \ \ \ \ \ \ \ \ \ \ k\ =\ (n\_bigram\ +\ 1)\ /\ BG\_SEG\_SZ\ +\ 1;}
\DoxyCodeLine{00394\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(\&k,\ \textcolor{keyword}{sizeof}(k),\ 1,\ fp)\ !=\ 1)}
\DoxyCodeLine{00395\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00396\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)\ SWAP\_INT32(\&k);}
\DoxyCodeLine{00397\ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>lm3g.tseg\_base\ =\ ckd\_calloc(k,\ \textcolor{keyword}{sizeof}(int32));}
\DoxyCodeLine{00398\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(model-\/>lm3g.tseg\_base,\ \textcolor{keyword}{sizeof}(int32),\ k,\ fp)\ !=}
\DoxyCodeLine{00399\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (\textcolor{keywordtype}{size\_t})\ k)\ \{}
\DoxyCodeLine{00400\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Failed\ to\ read\ trigram\ index\(\backslash\)n"{}});}
\DoxyCodeLine{00401\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00402\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00403\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)}
\DoxyCodeLine{00404\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ k;\ i++)}
\DoxyCodeLine{00405\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ SWAP\_INT32(\&model-\/>lm3g.tseg\_base[i]);}
\DoxyCodeLine{00406\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00407\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ LM.tseg\_base\ entries\ read\(\backslash\)n"{}},\ k);}
\DoxyCodeLine{00408\ \ \ \ \ \}}
\DoxyCodeLine{00409\ }
\DoxyCodeLine{00410\ \ \ \ \ \textcolor{comment}{/*\ read\ ascii\ word\ strings\ */}}
\DoxyCodeLine{00411\ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_mmap)\ \{}
\DoxyCodeLine{00412\ \ \ \ \ \ \ \ \ memcpy(\&k,\ map\_base\ +\ offset,\ \textcolor{keyword}{sizeof}(k));}
\DoxyCodeLine{00413\ \ \ \ \ \ \ \ \ offset\ +=\ \textcolor{keyword}{sizeof}(int32);}
\DoxyCodeLine{00414\ \ \ \ \ \ \ \ \ tmp\_word\_str\ =\ (\textcolor{keywordtype}{char}\ *)\ (map\_base\ +\ offset);}
\DoxyCodeLine{00415\ \ \ \ \ \ \ \ \ offset\ +=\ k;}
\DoxyCodeLine{00416\ \ \ \ \ \}}
\DoxyCodeLine{00417\ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00418\ \ \ \ \ \ \ \ \ base-\/>writable\ =\ TRUE;}
\DoxyCodeLine{00419\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(\&k,\ \textcolor{keyword}{sizeof}(k),\ 1,\ fp)\ !=\ 1)}
\DoxyCodeLine{00420\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00421\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_swap)\ SWAP\_INT32(\&k);}
\DoxyCodeLine{00422\ \ \ \ \ \ \ \ \ tmp\_word\_str\ =\ ckd\_calloc(k,\ 1);}
\DoxyCodeLine{00423\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fread(tmp\_word\_str,\ 1,\ k,\ fp)\ !=\ (\textcolor{keywordtype}{size\_t})\ k)\ \{}
\DoxyCodeLine{00424\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Failed\ to\ read\ words\(\backslash\)n"{}});}
\DoxyCodeLine{00425\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00426\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00427\ \ \ \ \ \}}
\DoxyCodeLine{00428\ }
\DoxyCodeLine{00429\ \ \ \ \ \textcolor{comment}{/*\ First\ make\ sure\ string\ just\ read\ contains\ n\_counts[0]\ words\ (PARANOIA!!)\ */}}
\DoxyCodeLine{00430\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0,\ j\ =\ 0;\ i\ <\ k;\ i++)}
\DoxyCodeLine{00431\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (tmp\_word\_str[i]\ ==\ \textcolor{charliteral}{'\(\backslash\)0'})}
\DoxyCodeLine{00432\ \ \ \ \ \ \ \ \ \ \ \ \ j++;}
\DoxyCodeLine{00433\ \ \ \ \ \textcolor{keywordflow}{if}\ (j\ !=\ n\_unigram)\ \{}
\DoxyCodeLine{00434\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Error\ reading\ word\ strings\ (\%d\ doesn't\ match\ n\_unigrams\ \%d)\(\backslash\)n"{}},}
\DoxyCodeLine{00435\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ j,\ n\_unigram);}
\DoxyCodeLine{00436\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{00437\ \ \ \ \ \}}
\DoxyCodeLine{00438\ }
\DoxyCodeLine{00439\ \ \ \ \ \textcolor{comment}{/*\ Break\ up\ string\ just\ read\ into\ words\ */}}
\DoxyCodeLine{00440\ \ \ \ \ \textcolor{keywordflow}{if}\ (do\_mmap)\ \{}
\DoxyCodeLine{00441\ \ \ \ \ \ \ \ \ j\ =\ 0;}
\DoxyCodeLine{00442\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ n\_unigram;\ i++)\ \{}
\DoxyCodeLine{00443\ \ \ \ \ \ \ \ \ \ \ \ \ base-\/>word\_str[i]\ =\ tmp\_word\_str\ +\ j;}
\DoxyCodeLine{00444\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (hash\_table\_enter(base-\/>wid,\ base-\/>word\_str[i],}
\DoxyCodeLine{00445\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (\textcolor{keywordtype}{void}\ *)(\textcolor{keywordtype}{long})i)\ !=\ (\textcolor{keywordtype}{void}\ *)(\textcolor{keywordtype}{long})i)\ \{}
\DoxyCodeLine{00446\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_WARN(\textcolor{stringliteral}{"{}Duplicate\ word\ in\ dictionary:\ \%s\(\backslash\)n"{}},\ base-\/>word\_str[i]);}
\DoxyCodeLine{00447\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00448\ \ \ \ \ \ \ \ \ \ \ \ \ j\ +=\ strlen(base-\/>word\_str[i])\ +\ 1;}
\DoxyCodeLine{00449\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00450\ \ \ \ \ \}}
\DoxyCodeLine{00451\ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00452\ \ \ \ \ \ \ \ \ j\ =\ 0;}
\DoxyCodeLine{00453\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ n\_unigram;\ i++)\ \{}
\DoxyCodeLine{00454\ \ \ \ \ \ \ \ \ \ \ \ \ base-\/>word\_str[i]\ =\ ckd\_salloc(tmp\_word\_str\ +\ j);}
\DoxyCodeLine{00455\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (hash\_table\_enter(base-\/>wid,\ base-\/>word\_str[i],}
\DoxyCodeLine{00456\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (\textcolor{keywordtype}{void}\ *)(\textcolor{keywordtype}{long})i)\ !=\ (\textcolor{keywordtype}{void}\ *)(\textcolor{keywordtype}{long})i)\ \{}
\DoxyCodeLine{00457\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_WARN(\textcolor{stringliteral}{"{}Duplicate\ word\ in\ dictionary:\ \%s\(\backslash\)n"{}},\ base-\/>word\_str[i]);}
\DoxyCodeLine{00458\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00459\ \ \ \ \ \ \ \ \ \ \ \ \ j\ +=\ strlen(base-\/>word\_str[i])\ +\ 1;}
\DoxyCodeLine{00460\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00461\ \ \ \ \ \ \ \ \ free(tmp\_word\_str);}
\DoxyCodeLine{00462\ \ \ \ \ \}}
\DoxyCodeLine{00463\ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ ascii\ word\ strings\ read\(\backslash\)n"{}},\ i);}
\DoxyCodeLine{00464\ }
\DoxyCodeLine{00465\ \ \ \ \ fclose\_comp(fp,\ is\_pipe);}
\DoxyCodeLine{00466\ \ \ \ \ \textcolor{keywordflow}{return}\ base;}
\DoxyCodeLine{00467\ }
\DoxyCodeLine{00468\ error\_out:}
\DoxyCodeLine{00469\ \ \ \ \ \textcolor{keywordflow}{if}\ (fp)}
\DoxyCodeLine{00470\ \ \ \ \ \ \ \ \ fclose\_comp(fp,\ is\_pipe);}
\DoxyCodeLine{00471\ \ \ \ \ ngram\_model\_free(base);}
\DoxyCodeLine{00472\ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00473\ \}}
\DoxyCodeLine{00474\ }
\DoxyCodeLine{00475\ ngram\_model\_dmp\_t\ *}
\DoxyCodeLine{00476\ ngram\_model\_dmp\_build(ngram\_model\_t\ *base)}
\DoxyCodeLine{00477\ \{}
\DoxyCodeLine{00478\ \ \ \ \ ngram\_model\_dmp\_t\ *model;}
\DoxyCodeLine{00479\ \ \ \ \ ngram\_model\_t\ *newbase;}
\DoxyCodeLine{00480\ \ \ \ \ ngram\_iter\_t\ *itor;}
\DoxyCodeLine{00481\ \ \ \ \ sorted\_list\_t\ sorted\_prob2;}
\DoxyCodeLine{00482\ \ \ \ \ sorted\_list\_t\ sorted\_bo\_wt2;}
\DoxyCodeLine{00483\ \ \ \ \ sorted\_list\_t\ sorted\_prob3;}
\DoxyCodeLine{00484\ \ \ \ \ bigram\_t\ *bgptr;}
\DoxyCodeLine{00485\ \ \ \ \ trigram\_t\ *tgptr;}
\DoxyCodeLine{00486\ \ \ \ \ \textcolor{keywordtype}{int}\ i,\ bgcount,\ tgcount,\ seg;}
\DoxyCodeLine{00487\ }
\DoxyCodeLine{00488\ \ \ \ \ \textcolor{keywordflow}{if}\ (base-\/>funcs\ ==\ \&ngram\_model\_dmp\_funcs)\ \{}
\DoxyCodeLine{00489\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}Using\ existing\ DMP\ model.\(\backslash\)n"{}});}
\DoxyCodeLine{00490\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ (ngram\_model\_dmp\_t\ *)ngram\_model\_retain(base);}
\DoxyCodeLine{00491\ \ \ \ \ \}}
\DoxyCodeLine{00492\ }
\DoxyCodeLine{00493\ \ \ \ \ \textcolor{comment}{/*\ Initialize\ new\ base\ model\ structure\ with\ params\ from\ base.\ */}}
\DoxyCodeLine{00494\ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}Building\ DMP\ model...\(\backslash\)n"{}});}
\DoxyCodeLine{00495\ \ \ \ \ model\ =\ ckd\_calloc(1,\ \textcolor{keyword}{sizeof}(*model));}
\DoxyCodeLine{00496\ \ \ \ \ newbase\ =\ \&model-\/>base;}
\DoxyCodeLine{00497\ \ \ \ \ ngram\_model\_init(newbase,\ \&ngram\_model\_dmp\_funcs,}
\DoxyCodeLine{00498\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ logmath\_retain(base-\/>lmath),}
\DoxyCodeLine{00499\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ base-\/>n,\ base-\/>n\_counts[0]);}
\DoxyCodeLine{00500\ \ \ \ \ \textcolor{comment}{/*\ Copy\ N-\/gram\ counts\ over.\ */}}
\DoxyCodeLine{00501\ \ \ \ \ memcpy(newbase-\/>n\_counts,\ base-\/>n\_counts,}
\DoxyCodeLine{00502\ \ \ \ \ \ \ \ \ \ \ \ base-\/>n\ *\ \textcolor{keyword}{sizeof}(*base-\/>n\_counts));}
\DoxyCodeLine{00503\ \ \ \ \ \textcolor{comment}{/*\ Make\ sure\ word\ strings\ are\ freed.\ */}}
\DoxyCodeLine{00504\ \ \ \ \ newbase-\/>writable\ =\ TRUE;}
\DoxyCodeLine{00505\ \ \ \ \ \textcolor{comment}{/*\ Initialize\ unigram\ table\ and\ string\ table.\ */}}
\DoxyCodeLine{00506\ \ \ \ \ model-\/>lm3g.unigrams\ =\ new\_unigram\_table(newbase-\/>n\_counts[0]\ +\ 1);}
\DoxyCodeLine{00507\ \ \ \ \ \textcolor{keywordflow}{for}\ (itor\ =\ ngram\_model\_mgrams(base,\ 0);\ itor;}
\DoxyCodeLine{00508\ \ \ \ \ \ \ \ \ \ itor\ =\ ngram\_iter\_next(itor))\ \{}
\DoxyCodeLine{00509\ \ \ \ \ \ \ \ \ int32\ prob1,\ bo\_wt1;}
\DoxyCodeLine{00510\ \ \ \ \ \ \ \ \ int32\ \textcolor{keyword}{const}\ *wids;}
\DoxyCodeLine{00511\ }
\DoxyCodeLine{00512\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Can't\ guarantee\ they\ will\ go\ in\ unigram\ order,\ so\ just\ to}}
\DoxyCodeLine{00513\ \textcolor{comment}{\ \ \ \ \ \ \ \ \ *\ be\ correct,\ we\ do\ this...\ */}}
\DoxyCodeLine{00514\ \ \ \ \ \ \ \ \ wids\ =\ ngram\_iter\_get(itor,\ \&prob1,\ \&bo\_wt1);}
\DoxyCodeLine{00515\ \ \ \ \ \ \ \ \ model-\/>lm3g.unigrams[wids[0]].prob1.l\ =\ prob1;}
\DoxyCodeLine{00516\ \ \ \ \ \ \ \ \ model-\/>lm3g.unigrams[wids[0]].bo\_wt1.l\ =\ bo\_wt1;}
\DoxyCodeLine{00517\ \ \ \ \ \ \ \ \ newbase-\/>word\_str[wids[0]]\ =\ ckd\_salloc(ngram\_word(base,\ wids[0]));}
\DoxyCodeLine{00518\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((hash\_table\_enter\_int32(newbase-\/>wid,}
\DoxyCodeLine{00519\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ newbase-\/>word\_str[wids[0]],\ wids[0]))}
\DoxyCodeLine{00520\ \ \ \ \ \ \ \ \ \ \ \ \ !=\ wids[0])\ \{}
\DoxyCodeLine{00521\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_WARN(\textcolor{stringliteral}{"{}Duplicate\ word\ in\ dictionary:\ \%s\(\backslash\)n"{}},\ newbase-\/>word\_str[wids[0]]);}
\DoxyCodeLine{00522\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00523\ \ \ \ \ \}}
\DoxyCodeLine{00524\ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ \#unigrams\ created\(\backslash\)n"{}},\ newbase-\/>n\_counts[0]);}
\DoxyCodeLine{00525\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }
\DoxyCodeLine{00526\ \ \ \ \ \textcolor{keywordflow}{if}\ (newbase-\/>n\ <\ 2)\ }
\DoxyCodeLine{00527\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ model;}
\DoxyCodeLine{00528\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }
\DoxyCodeLine{00529\ \ \ \ \ \textcolor{comment}{/*\ Construct\ quantized\ probability\ table\ for\ bigrams\ and}}
\DoxyCodeLine{00530\ \textcolor{comment}{\ \ \ \ \ *\ (optionally)\ trigrams.\ \ Hesitate\ to\ use\ the\ "{}sorted\ list"{}\ thing}}
\DoxyCodeLine{00531\ \textcolor{comment}{\ \ \ \ \ *\ since\ it\ isn't\ so\ useful,\ but\ it's\ there\ already.\ */}}
\DoxyCodeLine{00532\ \ \ \ \ init\_sorted\_list(\&sorted\_prob2);}
\DoxyCodeLine{00533\ \ \ \ \ \textcolor{keywordflow}{if}\ (newbase-\/>n\ >\ 2)\ \{}
\DoxyCodeLine{00534\ \ \ \ \ \ \ \ \ init\_sorted\_list(\&sorted\_bo\_wt2);}
\DoxyCodeLine{00535\ \ \ \ \ \ \ \ \ init\_sorted\_list(\&sorted\_prob3);}
\DoxyCodeLine{00536\ \ \ \ \ \}}
\DoxyCodeLine{00537\ \ \ \ \ \textcolor{comment}{/*\ Construct\ bigram\ and\ trigram\ arrays.\ */}}
\DoxyCodeLine{00538\ \ \ \ \ bgptr\ =\ model-\/>lm3g.bigrams\ =\ ckd\_calloc(newbase-\/>n\_counts[1]\ +\ 1,\ \textcolor{keyword}{sizeof}(bigram\_t));}
\DoxyCodeLine{00539\ \ \ \ \ \textcolor{keywordflow}{if}\ (newbase-\/>n\ >\ 2)\ \{}
\DoxyCodeLine{00540\ \ \ \ \ \ \ \ \ tgptr\ =\ model-\/>lm3g.trigrams\ =\ ckd\_calloc(newbase-\/>n\_counts[2],\ \textcolor{keyword}{sizeof}(trigram\_t));}
\DoxyCodeLine{00541\ \ \ \ \ \ \ \ \ model-\/>lm3g.tseg\_base\ =}
\DoxyCodeLine{00542\ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_calloc((newbase-\/>n\_counts[1]\ +\ 1)\ /\ BG\_SEG\_SZ\ +\ 1,\ \textcolor{keyword}{sizeof}(int32));}
\DoxyCodeLine{00543\ \ \ \ \ \}}
\DoxyCodeLine{00544\ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00545\ \ \ \ \ \ \ \ \ tgptr\ =\ NULL;}
\DoxyCodeLine{00546\ \ \ \ \ \textcolor{comment}{/*\ Since\ bigrams\ and\ trigrams\ have\ to\ be\ contiguous\ with\ others}}
\DoxyCodeLine{00547\ \textcolor{comment}{\ \ \ \ \ *\ with\ the\ same\ N-\/1-\/gram,\ we\ traverse\ them\ in\ depth-\/first\ order}}
\DoxyCodeLine{00548\ \textcolor{comment}{\ \ \ \ \ *\ to\ build\ the\ bigram\ and\ trigram\ arrays.\ */}}
\DoxyCodeLine{00549\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ newbase-\/>n\_counts[0];\ ++i)\ \{}
\DoxyCodeLine{00550\ \ \ \ \ \ \ \ \ ngram\_iter\_t\ *uitor;}
\DoxyCodeLine{00551\ \ \ \ \ \ \ \ \ bgcount\ =\ bgptr\ -\/\ model-\/>lm3g.bigrams;}
\DoxyCodeLine{00552\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ First\ bigram\ index\ (same\ as\ next\ if\ no\ bigrams...)\ */}}
\DoxyCodeLine{00553\ \ \ \ \ \ \ \ \ model-\/>lm3g.unigrams[i].bigrams\ =\ bgcount;}
\DoxyCodeLine{00554\ \ \ \ \ \ \ \ \ E\_DEBUG(2,\ (\textcolor{stringliteral}{"{}unigram\ \%d:\ \%s\ =>\ bigram\ \%d\(\backslash\)n"{}},\ i,\ newbase-\/>word\_str[i],\ bgcount));}
\DoxyCodeLine{00555\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ All\ bigrams\ corresponding\ to\ unigram\ i\ */}}
\DoxyCodeLine{00556\ \ \ \ \ \ \ \ \ uitor\ =\ ngram\_ng\_iter(base,\ i,\ NULL,\ 0);}
\DoxyCodeLine{00557\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (itor\ =\ ngram\_iter\_successors(uitor);}
\DoxyCodeLine{00558\ \ \ \ \ \ \ \ \ \ \ \ \ \ itor;\ ++bgptr,\ itor\ =\ ngram\_iter\_next(itor))\ \{}
\DoxyCodeLine{00559\ \ \ \ \ \ \ \ \ \ \ \ \ int32\ prob2,\ bo\_wt2;}
\DoxyCodeLine{00560\ \ \ \ \ \ \ \ \ \ \ \ \ int32\ \textcolor{keyword}{const}\ *wids;}
\DoxyCodeLine{00561\ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_iter\_t\ *titor;}
\DoxyCodeLine{00562\ }
\DoxyCodeLine{00563\ \ \ \ \ \ \ \ \ \ \ \ \ wids\ =\ ngram\_iter\_get(itor,\ \&prob2,\ \&bo\_wt2);}
\DoxyCodeLine{00564\ }
\DoxyCodeLine{00565\ \ \ \ \ \ \ \ \ \ \ \ \ assert\ (bgptr\ -\/\ model-\/>lm3g.bigrams\ <\ newbase-\/>n\_counts[1]);}
\DoxyCodeLine{00566\ }
\DoxyCodeLine{00567\ \ \ \ \ \ \ \ \ \ \ \ \ bgptr-\/>wid\ =\ wids[1];}
\DoxyCodeLine{00568\ \ \ \ \ \ \ \ \ \ \ \ \ bgptr-\/>prob2\ =\ sorted\_id(\&sorted\_prob2,\ \&prob2);}
\DoxyCodeLine{00569\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (newbase-\/>n\ >\ 2)\ \{}
\DoxyCodeLine{00570\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ tgcount\ =\ (tgptr\ -\/\ model-\/>lm3g.trigrams);}
\DoxyCodeLine{00571\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ bgcount\ =\ (bgptr\ -\/\ model-\/>lm3g.bigrams);}
\DoxyCodeLine{00572\ }
\DoxyCodeLine{00573\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Backoff\ weight\ (only\ if\ there\ are\ trigrams...)\ */}}
\DoxyCodeLine{00574\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ bgptr-\/>bo\_wt2\ =\ sorted\_id(\&sorted\_bo\_wt2,\ \&bo\_wt2);}
\DoxyCodeLine{00575\ }
\DoxyCodeLine{00576\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Find\ bigram\ segment\ for\ this\ bigram\ (this\ isn't}}
\DoxyCodeLine{00577\ \textcolor{comment}{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ *\ used\ unless\ there\ are\ trigrams)\ */}}
\DoxyCodeLine{00578\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ seg\ =\ bgcount\ >>\ LOG\_BG\_SEG\_SZ;}
\DoxyCodeLine{00579\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ If\ we\ just\ crossed\ a\ bigram\ segment\ boundary,\ then}}
\DoxyCodeLine{00580\ \textcolor{comment}{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ *\ point\ tseg\_base\ for\ the\ new\ segment\ to\ the\ current}}
\DoxyCodeLine{00581\ \textcolor{comment}{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ *\ trigram\ pointer.\ */}}
\DoxyCodeLine{00582\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (seg\ !=\ (bgcount\ -\/\ 1)\ >>\ LOG\_BG\_SEG\_SZ)}
\DoxyCodeLine{00583\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>lm3g.tseg\_base[seg]\ =\ tgcount;}
\DoxyCodeLine{00584\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Now\ calculate\ the\ trigram\ offset.\ */}}
\DoxyCodeLine{00585\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ bgptr-\/>trigrams\ =\ tgcount\ -\/\ model-\/>lm3g.tseg\_base[seg];}
\DoxyCodeLine{00586\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_DEBUG(2,\ (\textcolor{stringliteral}{"{}bigram\ \%d\ \%s\ \%s\ =>\ trigram\ \%d:\%d\(\backslash\)n"{}},}
\DoxyCodeLine{00587\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ bgcount,}
\DoxyCodeLine{00588\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ newbase-\/>word\_str[wids[0]],}
\DoxyCodeLine{00589\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ newbase-\/>word\_str[wids[1]],}
\DoxyCodeLine{00590\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ seg,\ bgptr-\/>trigrams));}
\DoxyCodeLine{00591\ }
\DoxyCodeLine{00592\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ And\ fill\ in\ successors'\ trigram\ info.\ */}}
\DoxyCodeLine{00593\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (titor\ =\ ngram\_iter\_successors(itor);}
\DoxyCodeLine{00594\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ titor;\ ++tgptr,\ titor\ =\ ngram\_iter\_next(titor))\ \{}
\DoxyCodeLine{00595\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ prob3,\ dummy;}
\DoxyCodeLine{00596\ }
\DoxyCodeLine{00597\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ assert(tgptr\ -\/\ model-\/>lm3g.trigrams\ <\ newbase-\/>n\_counts[2]);}
\DoxyCodeLine{00598\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ wids\ =\ ngram\_iter\_get(titor,\ \&prob3,\ \&dummy);}
\DoxyCodeLine{00599\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ tgptr-\/>wid\ =\ wids[2];}
\DoxyCodeLine{00600\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ tgptr-\/>prob3\ =\ sorted\_id(\&sorted\_prob3,\ \&prob3);}
\DoxyCodeLine{00601\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_DEBUG(2,\ (\textcolor{stringliteral}{"{}trigram\ \%d\ \%s\ \%s\ \%s\ =>\ prob\ \%d\(\backslash\)n"{}},}
\DoxyCodeLine{00602\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ tgcount,}
\DoxyCodeLine{00603\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ newbase-\/>word\_str[wids[0]],}
\DoxyCodeLine{00604\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ newbase-\/>word\_str[wids[1]],}
\DoxyCodeLine{00605\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ newbase-\/>word\_str[wids[2]],}
\DoxyCodeLine{00606\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ tgptr-\/>prob3));}
\DoxyCodeLine{00607\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00608\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00609\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00610\ \ \ \ \ \ \ \ \ ngram\_iter\_free(uitor);}
\DoxyCodeLine{00611\ \ \ \ \ \}}
\DoxyCodeLine{00612\ \ \ \ \ \textcolor{comment}{/*\ Add\ sentinal\ unigram\ and\ bigram\ records.\ */}}
\DoxyCodeLine{00613\ \ \ \ \ bgcount\ =\ bgptr\ -\/\ model-\/>lm3g.bigrams;}
\DoxyCodeLine{00614\ \ \ \ \ tgcount\ =\ tgptr\ -\/\ model-\/>lm3g.trigrams;}
\DoxyCodeLine{00615\ \ \ \ \ seg\ =\ bgcount\ >>\ LOG\_BG\_SEG\_SZ;}
\DoxyCodeLine{00616\ \ \ \ \ \textcolor{keywordflow}{if}\ (seg\ !=\ (bgcount\ -\/\ 1)\ >>\ LOG\_BG\_SEG\_SZ)}
\DoxyCodeLine{00617\ \ \ \ \ \ \ \ \ model-\/>lm3g.tseg\_base[seg]\ =\ tgcount;}
\DoxyCodeLine{00618\ \ \ \ \ model-\/>lm3g.unigrams[i].bigrams\ =\ bgcount;}
\DoxyCodeLine{00619\ \ \ \ \ \textcolor{keywordflow}{if}\ (newbase-\/>n\ >\ 2)}
\DoxyCodeLine{00620\ \ \ \ \ \ \ \ \ bgptr-\/>trigrams\ =\ tgcount\ -\/\ model-\/>lm3g.tseg\_base[seg];}
\DoxyCodeLine{00621\ }
\DoxyCodeLine{00622\ \ \ \ \ \textcolor{comment}{/*\ Now\ create\ probability\ tables.\ */}}
\DoxyCodeLine{00623\ \ \ \ \ model-\/>lm3g.n\_prob2\ =\ sorted\_prob2.free;}
\DoxyCodeLine{00624\ \ \ \ \ model-\/>lm3g.prob2\ =\ vals\_in\_sorted\_list(\&sorted\_prob2);}
\DoxyCodeLine{00625\ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ \#bigrams\ created\(\backslash\)n"{}},\ newbase-\/>n\_counts[1]);}
\DoxyCodeLine{00626\ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ \#prob2\ entries\(\backslash\)n"{}},\ model-\/>lm3g.n\_prob2);}
\DoxyCodeLine{00627\ \ \ \ \ free\_sorted\_list(\&sorted\_prob2);}
\DoxyCodeLine{00628\ \ \ \ \ \textcolor{keywordflow}{if}\ (newbase-\/>n\ >\ 2)\ \{}
\DoxyCodeLine{00629\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Create\ trigram\ bo-\/wts\ array.\ */}}
\DoxyCodeLine{00630\ \ \ \ \ \ \ \ \ model-\/>lm3g.n\_bo\_wt2\ =\ sorted\_bo\_wt2.free;}
\DoxyCodeLine{00631\ \ \ \ \ \ \ \ \ model-\/>lm3g.bo\_wt2\ =\ vals\_in\_sorted\_list(\&sorted\_bo\_wt2);}
\DoxyCodeLine{00632\ \ \ \ \ \ \ \ \ free\_sorted\_list(\&sorted\_bo\_wt2);}
\DoxyCodeLine{00633\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ \#bo\_wt2\ entries\(\backslash\)n"{}},\ model-\/>lm3g.n\_bo\_wt2);}
\DoxyCodeLine{00634\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Create\ trigram\ probability\ table.\ */}}
\DoxyCodeLine{00635\ \ \ \ \ \ \ \ \ model-\/>lm3g.n\_prob3\ =\ sorted\_prob3.free;}
\DoxyCodeLine{00636\ \ \ \ \ \ \ \ \ model-\/>lm3g.prob3\ =\ vals\_in\_sorted\_list(\&sorted\_prob3);}
\DoxyCodeLine{00637\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ \#trigrams\ created\(\backslash\)n"{}},\ newbase-\/>n\_counts[2]);}
\DoxyCodeLine{00638\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ \#prob3\ entries\(\backslash\)n"{}},\ model-\/>lm3g.n\_prob3);}
\DoxyCodeLine{00639\ \ \ \ \ \ \ \ \ free\_sorted\_list(\&sorted\_prob3);}
\DoxyCodeLine{00640\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Initialize\ tginfo\ */}}
\DoxyCodeLine{00641\ \ \ \ \ \ \ \ \ model-\/>lm3g.tginfo\ =\ ckd\_calloc(newbase-\/>n\_counts[0],\ \textcolor{keyword}{sizeof}(tginfo\_t\ *));}
\DoxyCodeLine{00642\ \ \ \ \ \ \ \ \ model-\/>lm3g.le\ =\ listelem\_alloc\_init(\textcolor{keyword}{sizeof}(tginfo\_t));}
\DoxyCodeLine{00643\ \ \ \ \ \}}
\DoxyCodeLine{00644\ }
\DoxyCodeLine{00645\ \ \ \ \ \textcolor{keywordflow}{return}\ model;}
\DoxyCodeLine{00646\ \}}
\DoxyCodeLine{00647\ }
\DoxyCodeLine{00648\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00649\ fwrite\_int32(FILE\ *fh,\ int32\ val)}
\DoxyCodeLine{00650\ \{}
\DoxyCodeLine{00651\ \ \ \ \ fwrite(\&val,\ 4,\ 1,\ fh);}
\DoxyCodeLine{00652\ \}}
\DoxyCodeLine{00653\ }
\DoxyCodeLine{00654\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00655\ fwrite\_ug(FILE\ *fh,\ unigram\_t\ *ug,\ logmath\_t\ *lmath)}
\DoxyCodeLine{00656\ \{}
\DoxyCodeLine{00657\ \ \ \ \ int32\ bogus\ =\ -\/1;}
\DoxyCodeLine{00658\ \ \ \ \ float32\ log10val;}
\DoxyCodeLine{00659\ }
\DoxyCodeLine{00660\ \ \ \ \ \textcolor{comment}{/*\ Bogus\ dictionary\ mapping\ field.\ */}}
\DoxyCodeLine{00661\ \ \ \ \ fwrite(\&bogus,\ 4,\ 1,\ fh);}
\DoxyCodeLine{00662\ \ \ \ \ \textcolor{comment}{/*\ Convert\ values\ to\ log10.\ */}}
\DoxyCodeLine{00663\ \ \ \ \ log10val\ =\ logmath\_log\_to\_log10(lmath,\ ug-\/>prob1.l);}
\DoxyCodeLine{00664\ \ \ \ \ fwrite(\&log10val,\ 4,\ 1,\ fh);}
\DoxyCodeLine{00665\ \ \ \ \ log10val\ =\ logmath\_log\_to\_log10(lmath,\ ug-\/>bo\_wt1.l);}
\DoxyCodeLine{00666\ \ \ \ \ fwrite(\&log10val,\ 4,\ 1,\ fh);}
\DoxyCodeLine{00667\ \ \ \ \ fwrite\_int32(fh,\ ug-\/>bigrams);}
\DoxyCodeLine{00668\ \}}
\DoxyCodeLine{00669\ }
\DoxyCodeLine{00670\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00671\ fwrite\_bg(FILE\ *fh,\ bigram\_t\ *bg)}
\DoxyCodeLine{00672\ \{}
\DoxyCodeLine{00673\ \ \ \ \ fwrite(bg,\ \textcolor{keyword}{sizeof}(*bg),\ 1,\ fh);}
\DoxyCodeLine{00674\ \}}
\DoxyCodeLine{00675\ }
\DoxyCodeLine{00676\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00677\ fwrite\_tg(FILE\ *fh,\ trigram\_t\ *tg)}
\DoxyCodeLine{00678\ \{}
\DoxyCodeLine{00679\ \ \ \ \ fwrite(tg,\ \textcolor{keyword}{sizeof}(*tg),\ 1,\ fh);}
\DoxyCodeLine{00680\ \}}
\DoxyCodeLine{00681\ }
\DoxyCodeLine{00684\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{char}\ \textcolor{keyword}{const}\ *fmtdesc[]\ =\ \{}
\DoxyCodeLine{00685\ \ \ \ \ \textcolor{stringliteral}{"{}BEGIN\ FILE\ FORMAT\ DESCRIPTION"{}},}
\DoxyCodeLine{00686\ \ \ \ \ \textcolor{stringliteral}{"{}Header\ string\ length\ (int32)\ and\ string\ (including\ trailing\ 0)"{}},}
\DoxyCodeLine{00687\ \ \ \ \ \textcolor{stringliteral}{"{}Original\ LM\ filename\ string-\/length\ (int32)\ and\ filename\ (including\ trailing\ 0)"{}},}
\DoxyCodeLine{00688\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ version\ number\ (present\ iff\ value\ <=\ 0)"{}},}
\DoxyCodeLine{00689\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ original\ LM\ file\ modification\ timestamp\ (iff\ version\#\ present)"{}},}
\DoxyCodeLine{00690\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ string-\/length\ and\ string\ (including\ trailing\ 0)\ (iff\ version\#\ present)"{}},}
\DoxyCodeLine{00691\ \ \ \ \ \textcolor{stringliteral}{"{}...\ previous\ entry\ continued\ any\ number\ of\ times\ (iff\ version\#\ present)"{}},}
\DoxyCodeLine{00692\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ 0\ (terminating\ sequence\ of\ strings)\ (iff\ version\#\ present)"{}},}
\DoxyCodeLine{00693\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ log\_bg\_seg\_sz\ (present\ iff\ different\ from\ default\ value\ of\ LOG2\_BG\_SEG\_SZ)"{}},}
\DoxyCodeLine{00694\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ lm\_t.ucount\ (must\ be\ >\ 0)"{}},}
\DoxyCodeLine{00695\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ lm\_t.bcount"{}},}
\DoxyCodeLine{00696\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ lm\_t.tcount"{}},}
\DoxyCodeLine{00697\ \ \ \ \ \textcolor{stringliteral}{"{}lm\_t.ucount+1\ unigrams\ (including\ sentinel)"{}},}
\DoxyCodeLine{00698\ \ \ \ \ \textcolor{stringliteral}{"{}lm\_t.bcount+1\ bigrams\ (including\ sentinel\ 64\ bits\ (bg\_t)\ each\ if\ version=-\/1/-\/2,\ 128\ bits\ (bg32\_t)\ each\ if\ version=-\/3"{}},}
\DoxyCodeLine{00699\ \ \ \ \ \textcolor{stringliteral}{"{}lm\_t.tcount\ trigrams\ (present\ iff\ lm\_t.tcount\ >\ 0\ 32\ bits\ (tg\_t)\ each\ if\ version=-\/1/-\/2,\ 64\ bits\ (tg32\_t)\ each\ if\ version=-\/3)"{}},}
\DoxyCodeLine{00700\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ lm\_t.n\_prob2"{}},}
\DoxyCodeLine{00701\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ lm\_t.prob2[]"{}},}
\DoxyCodeLine{00702\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ lm\_t.n\_bo\_wt2\ (present\ iff\ lm\_t.tcount\ >\ 0)"{}},}
\DoxyCodeLine{00703\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ lm\_t.bo\_wt2[]\ (present\ iff\ lm\_t.tcount\ >\ 0)"{}},}
\DoxyCodeLine{00704\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ lm\_t.n\_prob3\ (present\ iff\ lm\_t.tcount\ >\ 0)"{}},}
\DoxyCodeLine{00705\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ lm\_t.prob3[]\ (present\ iff\ lm\_t.tcount\ >\ 0)"{}},}
\DoxyCodeLine{00706\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ (lm\_t.bcount+1)/BG\_SEG\_SZ+1\ (present\ iff\ lm\_t.tcount\ >\ 0)"{}},}
\DoxyCodeLine{00707\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ lm\_t.tseg\_base[]\ (present\ iff\ lm\_t.tcount\ >\ 0)"{}},}
\DoxyCodeLine{00708\ \ \ \ \ \textcolor{stringliteral}{"{}(int32)\ Sum(all\ word\ string-\/lengths,\ including\ trailing\ 0\ for\ each)"{}},}
\DoxyCodeLine{00709\ \ \ \ \ \textcolor{stringliteral}{"{}All\ word\ strings\ (including\ trailing\ 0\ for\ each)"{}},}
\DoxyCodeLine{00710\ \ \ \ \ \textcolor{stringliteral}{"{}END\ FILE\ FORMAT\ DESCRIPTION"{}},}
\DoxyCodeLine{00711\ \ \ \ \ NULL,}
\DoxyCodeLine{00712\ \};}
\DoxyCodeLine{00713\ }
\DoxyCodeLine{00714\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00715\ ngram\_model\_dmp\_write\_header(FILE\ *\ fh)}
\DoxyCodeLine{00716\ \{}
\DoxyCodeLine{00717\ \ \ \ \ int32\ k;}
\DoxyCodeLine{00718\ \ \ \ \ k\ =\ strlen(darpa\_hdr)\ +\ 1;}
\DoxyCodeLine{00719\ \ \ \ \ fwrite\_int32(fh,\ k);}
\DoxyCodeLine{00720\ \ \ \ \ fwrite(darpa\_hdr,\ 1,\ k,\ fh);}
\DoxyCodeLine{00721\ \}}
\DoxyCodeLine{00722\ }
\DoxyCodeLine{00723\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00724\ ngram\_model\_dmp\_write\_lm\_filename(FILE\ *\ fh,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *lmfile)}
\DoxyCodeLine{00725\ \{}
\DoxyCodeLine{00726\ \ \ \ \ int32\ k;}
\DoxyCodeLine{00727\ }
\DoxyCodeLine{00728\ \ \ \ \ k\ =\ strlen(lmfile)\ +\ 1;}
\DoxyCodeLine{00729\ \ \ \ \ fwrite\_int32(fh,\ k);}
\DoxyCodeLine{00730\ \ \ \ \ fwrite(lmfile,\ 1,\ k,\ fh);}
\DoxyCodeLine{00731\ \}}
\DoxyCodeLine{00732\ }
\DoxyCodeLine{00733\ \textcolor{preprocessor}{\#define\ LMDMP\_VERSION\_TG\_16BIT\ -\/1\ }}
\DoxyCodeLine{00737\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00738\ ngram\_model\_dmp\_write\_version(FILE\ *\ fh,\ int32\ mtime)}
\DoxyCodeLine{00739\ \{}
\DoxyCodeLine{00740\ \ \ \ \ fwrite\_int32(fh,\ LMDMP\_VERSION\_TG\_16BIT);\ \ \ \textcolor{comment}{/*\ version\ \#\ */}}
\DoxyCodeLine{00741\ \ \ \ \ fwrite\_int32(fh,\ mtime);}
\DoxyCodeLine{00742\ \}}
\DoxyCodeLine{00743\ }
\DoxyCodeLine{00744\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00745\ ngram\_model\_dmp\_write\_ngram\_counts(FILE\ *\ fh,\ ngram\_model\_t\ *model)}
\DoxyCodeLine{00746\ \{}
\DoxyCodeLine{00747\ \ \ \ \ fwrite\_int32(fh,\ model-\/>n\_counts[0]);}
\DoxyCodeLine{00748\ \ \ \ \ fwrite\_int32(fh,\ model-\/>n\_counts[1]);}
\DoxyCodeLine{00749\ \ \ \ \ fwrite\_int32(fh,\ model-\/>n\_counts[2]);}
\DoxyCodeLine{00750\ \}}
\DoxyCodeLine{00751\ }
\DoxyCodeLine{00752\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00753\ ngram\_model\_dmp\_write\_fmtdesc(FILE\ *\ fh)}
\DoxyCodeLine{00754\ \{}
\DoxyCodeLine{00755\ \ \ \ \ int32\ i,\ k;}
\DoxyCodeLine{00756\ \ \ \ \ \textcolor{keywordtype}{long}\ pos;}
\DoxyCodeLine{00757\ }
\DoxyCodeLine{00758\ \ \ \ \ \textcolor{comment}{/*\ Write\ file\ format\ description\ into\ header\ */}}
\DoxyCodeLine{00759\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ fmtdesc[i]\ !=\ NULL;\ i++)\ \{}
\DoxyCodeLine{00760\ \ \ \ \ \ \ \ \ k\ =\ strlen(fmtdesc[i])\ +\ 1;}
\DoxyCodeLine{00761\ \ \ \ \ \ \ \ \ fwrite\_int32(fh,\ k);}
\DoxyCodeLine{00762\ \ \ \ \ \ \ \ \ fwrite(fmtdesc[i],\ 1,\ k,\ fh);}
\DoxyCodeLine{00763\ \ \ \ \ \}}
\DoxyCodeLine{00764\ \ \ \ \ \textcolor{comment}{/*\ Pad\ it\ out\ in\ order\ to\ achieve\ 32-\/bit\ alignment\ */}}
\DoxyCodeLine{00765\ \ \ \ \ pos\ =\ ftell(fh);}
\DoxyCodeLine{00766\ \ \ \ \ k\ =\ pos\ \&\ 3;}
\DoxyCodeLine{00767\ \ \ \ \ \textcolor{keywordflow}{if}\ (k)\ \{}
\DoxyCodeLine{00768\ \ \ \ \ \ \ \ \ fwrite\_int32(fh,\ 4-\/k);}
\DoxyCodeLine{00769\ \ \ \ \ \ \ \ \ fwrite(\textcolor{stringliteral}{"{}!!!!"{}},\ 1,\ 4-\/k,\ fh);}
\DoxyCodeLine{00770\ \ \ \ \ \}}
\DoxyCodeLine{00771\ \ \ \ \ fwrite\_int32(fh,\ 0);}
\DoxyCodeLine{00772\ \}}
\DoxyCodeLine{00773\ }
\DoxyCodeLine{00774\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00775\ ngram\_model\_dmp\_write\_unigram(FILE\ *fh,\ ngram\_model\_t\ *model)}
\DoxyCodeLine{00776\ \{}
\DoxyCodeLine{00777\ \ \ \ \ ngram\_model\_dmp\_t\ *lm\ =\ (ngram\_model\_dmp\_t\ *)model;}
\DoxyCodeLine{00778\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00779\ }
\DoxyCodeLine{00780\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <=\ model-\/>n\_counts[0];\ i++)\ \{}
\DoxyCodeLine{00781\ \ \ \ \ \ \ \ \ fwrite\_ug(fh,\ \&(lm-\/>lm3g.unigrams[i]),\ model-\/>lmath);}
\DoxyCodeLine{00782\ \ \ \ \ \}}
\DoxyCodeLine{00783\ \}}
\DoxyCodeLine{00784\ }
\DoxyCodeLine{00785\ }
\DoxyCodeLine{00786\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00787\ ngram\_model\_dmp\_write\_bigram(FILE\ *fh,\ ngram\_model\_t\ *model)}
\DoxyCodeLine{00788\ \{}
\DoxyCodeLine{00789\ \ \ \ \ ngram\_model\_dmp\_t\ *lm\ =\ (ngram\_model\_dmp\_t\ *)model;}
\DoxyCodeLine{00790\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00791\ }
\DoxyCodeLine{00792\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <=\ model-\/>n\_counts[1];\ i++)\ \{}
\DoxyCodeLine{00793\ \ \ \ \ \ \ \ \ fwrite\_bg(fh,\ \&(lm-\/>lm3g.bigrams[i]));}
\DoxyCodeLine{00794\ \ \ \ \ \}}
\DoxyCodeLine{00795\ }
\DoxyCodeLine{00796\ \}}
\DoxyCodeLine{00797\ }
\DoxyCodeLine{00798\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00799\ ngram\_model\_dmp\_write\_trigram(FILE\ *fh,\ ngram\_model\_t\ *model)}
\DoxyCodeLine{00800\ \{}
\DoxyCodeLine{00801\ \ \ \ \ ngram\_model\_dmp\_t\ *lm\ =\ (ngram\_model\_dmp\_t\ *)model;}
\DoxyCodeLine{00802\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00803\ }
\DoxyCodeLine{00804\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ model-\/>n\_counts[2];\ i++)\ \{}
\DoxyCodeLine{00805\ \ \ \ \ \ \ \ \ fwrite\_tg(fh,\ \&(lm-\/>lm3g.trigrams[i]));}
\DoxyCodeLine{00806\ \ \ \ \ \}}
\DoxyCodeLine{00807\ \}}
\DoxyCodeLine{00808\ }
\DoxyCodeLine{00809\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00810\ ngram\_model\_dmp\_write\_bgprob(FILE\ *fh,\ ngram\_model\_t\ *model)}
\DoxyCodeLine{00811\ \{}
\DoxyCodeLine{00812\ \ \ \ \ ngram\_model\_dmp\_t\ *lm\ =\ (ngram\_model\_dmp\_t\ *)model;}
\DoxyCodeLine{00813\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00814\ }
\DoxyCodeLine{00815\ \ \ \ \ fwrite\_int32(fh,\ lm-\/>lm3g.n\_prob2);}
\DoxyCodeLine{00816\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ lm-\/>lm3g.n\_prob2;\ i++)\ \{}
\DoxyCodeLine{00817\ \ \ \ \ \ \ \ \ float32\ log10val\ =\ logmath\_log\_to\_log10(model-\/>lmath,\ lm-\/>lm3g.prob2[i].l);}
\DoxyCodeLine{00818\ \ \ \ \ \ \ \ \ fwrite(\&log10val,\ 4,\ 1,\ fh);}
\DoxyCodeLine{00819\ \ \ \ \ \}}
\DoxyCodeLine{00820\ \}}
\DoxyCodeLine{00821\ }
\DoxyCodeLine{00822\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00823\ ngram\_model\_dmp\_write\_tgbowt(FILE\ *fh,\ ngram\_model\_t\ *model)}
\DoxyCodeLine{00824\ \{}
\DoxyCodeLine{00825\ \ \ \ \ ngram\_model\_dmp\_t\ *lm\ =\ (ngram\_model\_dmp\_t\ *)model;}
\DoxyCodeLine{00826\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00827\ }
\DoxyCodeLine{00828\ \ \ \ \ fwrite\_int32(fh,\ lm-\/>lm3g.n\_bo\_wt2);}
\DoxyCodeLine{00829\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ lm-\/>lm3g.n\_bo\_wt2;\ i++)\ \{}
\DoxyCodeLine{00830\ \ \ \ \ \ \ \ \ float32\ log10val\ =\ logmath\_log\_to\_log10(model-\/>lmath,\ lm-\/>lm3g.bo\_wt2[i].l);}
\DoxyCodeLine{00831\ \ \ \ \ \ \ \ \ fwrite(\&log10val,\ 4,\ 1,\ fh);}
\DoxyCodeLine{00832\ \ \ \ \ \}}
\DoxyCodeLine{00833\ \}}
\DoxyCodeLine{00834\ }
\DoxyCodeLine{00835\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00836\ ngram\_model\_dmp\_write\_tgprob(FILE\ *fh,\ ngram\_model\_t\ *model)}
\DoxyCodeLine{00837\ \{}
\DoxyCodeLine{00838\ \ \ \ \ ngram\_model\_dmp\_t\ *lm\ =\ (ngram\_model\_dmp\_t\ *)model;}
\DoxyCodeLine{00839\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00840\ }
\DoxyCodeLine{00841\ \ \ \ \ fwrite\_int32(fh,\ lm-\/>lm3g.n\_prob3);}
\DoxyCodeLine{00842\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ lm-\/>lm3g.n\_prob3;\ i++)\ \{}
\DoxyCodeLine{00843\ \ \ \ \ \ \ \ \ float32\ log10val\ =\ logmath\_log\_to\_log10(model-\/>lmath,\ lm-\/>lm3g.prob3[i].l);}
\DoxyCodeLine{00844\ \ \ \ \ \ \ \ \ fwrite(\&log10val,\ 4,\ 1,\ fh);}
\DoxyCodeLine{00845\ \ \ \ \ \}}
\DoxyCodeLine{00846\ \}}
\DoxyCodeLine{00847\ }
\DoxyCodeLine{00848\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00849\ ngram\_model\_dmp\_write\_tg\_segbase(FILE\ *fh,\ ngram\_model\_t\ *model)}
\DoxyCodeLine{00850\ \{}
\DoxyCodeLine{00851\ \ \ \ \ ngram\_model\_dmp\_t\ *lm\ =\ (ngram\_model\_dmp\_t\ *)model;}
\DoxyCodeLine{00852\ \ \ \ \ int32\ i,\ k;}
\DoxyCodeLine{00853\ }
\DoxyCodeLine{00854\ \ \ \ \ k\ =\ (model-\/>n\_counts[1]\ +\ 1)\ /\ BG\_SEG\_SZ\ +\ 1;}
\DoxyCodeLine{00855\ \ \ \ \ fwrite\_int32(fh,\ k);}
\DoxyCodeLine{00856\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ k;\ i++)}
\DoxyCodeLine{00857\ \ \ \ \ \ \ \ \ fwrite\_int32(fh,\ lm-\/>lm3g.tseg\_base[i]);}
\DoxyCodeLine{00858\ \}}
\DoxyCodeLine{00859\ }
\DoxyCodeLine{00860\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00861\ ngram\_model\_dmp\_write\_wordstr(FILE\ *fh,\ ngram\_model\_t\ *model)}
\DoxyCodeLine{00862\ \{}
\DoxyCodeLine{00863\ \ \ \ \ int32\ i,\ k;}
\DoxyCodeLine{00864\ }
\DoxyCodeLine{00865\ \ \ \ \ k\ =\ 0;}
\DoxyCodeLine{00866\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ model-\/>n\_counts[0];\ i++)}
\DoxyCodeLine{00867\ \ \ \ \ \ \ \ \ k\ +=\ strlen(model-\/>word\_str[i])\ +\ 1;}
\DoxyCodeLine{00868\ \ \ \ \ fwrite\_int32(fh,\ k);}
\DoxyCodeLine{00869\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ model-\/>n\_counts[0];\ i++)}
\DoxyCodeLine{00870\ \ \ \ \ \ \ \ \ fwrite(model-\/>word\_str[i],\ 1,}
\DoxyCodeLine{00871\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ strlen(model-\/>word\_str[i])\ +\ 1,\ fh);}
\DoxyCodeLine{00872\ \}}
\DoxyCodeLine{00873\ }
\DoxyCodeLine{00874\ \textcolor{keywordtype}{int}}
\DoxyCodeLine{00875\ ngram\_model\_dmp\_write(ngram\_model\_t\ *base,}
\DoxyCodeLine{00876\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *file\_name)}
\DoxyCodeLine{00877\ \{}
\DoxyCodeLine{00878\ \ \ \ \ ngram\_model\_dmp\_t\ *model;}
\DoxyCodeLine{00879\ \ \ \ \ ngram\_model\_t\ *newbase;}
\DoxyCodeLine{00880\ \ \ \ \ FILE\ *fh;}
\DoxyCodeLine{00881\ }
\DoxyCodeLine{00882\ \ \ \ \ \textcolor{comment}{/*\ First,\ construct\ a\ DMP\ model\ from\ the\ base\ model.\ */}}
\DoxyCodeLine{00883\ \ \ \ \ model\ =\ ngram\_model\_dmp\_build(base);}
\DoxyCodeLine{00884\ \ \ \ \ newbase\ =\ \&model-\/>base;}
\DoxyCodeLine{00885\ }
\DoxyCodeLine{00886\ \ \ \ \ \textcolor{comment}{/*\ Now\ write\ it,\ confident\ in\ the\ knowledge\ that\ it's\ the\ right}}
\DoxyCodeLine{00887\ \textcolor{comment}{\ \ \ \ \ *\ kind\ of\ language\ model\ internally.\ */}}
\DoxyCodeLine{00888\ \ \ \ \ \textcolor{keywordflow}{if}\ ((fh\ =\ fopen(file\_name,\ \textcolor{stringliteral}{"{}wb"{}}))\ ==\ NULL)\ \{}
\DoxyCodeLine{00889\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Cannot\ create\ file\ \%s\(\backslash\)n"{}},\ file\_name);}
\DoxyCodeLine{00890\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00891\ \ \ \ \ \}}
\DoxyCodeLine{00892\ \ \ \ \ ngram\_model\_dmp\_write\_header(fh);}
\DoxyCodeLine{00893\ \ \ \ \ ngram\_model\_dmp\_write\_lm\_filename(fh,\ file\_name);}
\DoxyCodeLine{00894\ \ \ \ \ ngram\_model\_dmp\_write\_version(fh,\ 0);}
\DoxyCodeLine{00895\ \ \ \ \ ngram\_model\_dmp\_write\_fmtdesc(fh);}
\DoxyCodeLine{00896\ \ \ \ \ ngram\_model\_dmp\_write\_ngram\_counts(fh,\ newbase);}
\DoxyCodeLine{00897\ \ \ \ \ ngram\_model\_dmp\_write\_unigram(fh,\ newbase);}
\DoxyCodeLine{00898\ \ \ \ \ \textcolor{keywordflow}{if}\ (newbase-\/>n\ >\ 1)\ \{}
\DoxyCodeLine{00899\ \ \ \ \ \ \ \ \ ngram\_model\_dmp\_write\_bigram(fh,\ newbase);}
\DoxyCodeLine{00900\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (newbase-\/>n\ >\ 2)\ \{}
\DoxyCodeLine{00901\ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_model\_dmp\_write\_trigram(fh,\ newbase);}
\DoxyCodeLine{00902\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00903\ \ \ \ \ \ \ \ \ ngram\_model\_dmp\_write\_bgprob(fh,\ newbase);}
\DoxyCodeLine{00904\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (newbase-\/>n\ >\ 2)\ \{}
\DoxyCodeLine{00905\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_model\_dmp\_write\_tgbowt(fh,\ newbase);}
\DoxyCodeLine{00906\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_model\_dmp\_write\_tgprob(fh,\ newbase);}
\DoxyCodeLine{00907\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_model\_dmp\_write\_tg\_segbase(fh,\ newbase);}
\DoxyCodeLine{00908\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00909\ \ \ \ \ \}}
\DoxyCodeLine{00910\ \ \ \ \ ngram\_model\_dmp\_write\_wordstr(fh,\ newbase);}
\DoxyCodeLine{00911\ \ \ \ \ ngram\_model\_free(newbase);}
\DoxyCodeLine{00912\ }
\DoxyCodeLine{00913\ \ \ \ \ \textcolor{keywordflow}{return}\ fclose(fh);}
\DoxyCodeLine{00914\ \}}
\DoxyCodeLine{00915\ }
\DoxyCodeLine{00916\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{int}}
\DoxyCodeLine{00917\ ngram\_model\_dmp\_apply\_weights(ngram\_model\_t\ *base,\ float32\ lw,}
\DoxyCodeLine{00918\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ float32\ wip,\ float32\ uw)}
\DoxyCodeLine{00919\ \{}
\DoxyCodeLine{00920\ \ \ \ \ ngram\_model\_dmp\_t\ *model\ =\ (ngram\_model\_dmp\_t\ *)base;}
\DoxyCodeLine{00921\ \ \ \ \ lm3g\_apply\_weights(base,\ \&model-\/>lm3g,\ lw,\ wip,\ uw);}
\DoxyCodeLine{00922\ \ \ \ \ \textcolor{keywordflow}{return}\ 0;}
\DoxyCodeLine{00923\ \}}
\DoxyCodeLine{00924\ }
\DoxyCodeLine{00925\ \textcolor{comment}{/*\ Lousy\ "{}templating"{}\ for\ things\ that\ are\ largely\ the\ same\ in\ DMP\ and}}
\DoxyCodeLine{00926\ \textcolor{comment}{\ *\ ARPA\ models,\ except\ for\ the\ bigram\ and\ trigram\ types\ and\ some}}
\DoxyCodeLine{00927\ \textcolor{comment}{\ *\ names.\ */}}
\DoxyCodeLine{00928\ \textcolor{preprocessor}{\#define\ NGRAM\_MODEL\_TYPE\ ngram\_model\_dmp\_t}}
\DoxyCodeLine{00929\ \textcolor{preprocessor}{\#include\ "{}lm3g\_templates.c"{}}}
\DoxyCodeLine{00930\ }
\DoxyCodeLine{00931\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00932\ ngram\_model\_dmp\_free(ngram\_model\_t\ *base)}
\DoxyCodeLine{00933\ \{}
\DoxyCodeLine{00934\ \ \ \ \ ngram\_model\_dmp\_t\ *model\ =\ (ngram\_model\_dmp\_t\ *)base;}
\DoxyCodeLine{00935\ }
\DoxyCodeLine{00936\ \ \ \ \ ckd\_free(model-\/>lm3g.unigrams);}
\DoxyCodeLine{00937\ \ \ \ \ ckd\_free(model-\/>lm3g.prob2);}
\DoxyCodeLine{00938\ \ \ \ \ \textcolor{keywordflow}{if}\ (model-\/>dump\_mmap)\ \{}
\DoxyCodeLine{00939\ \ \ \ \ \ \ \ \ mmio\_file\_unmap(model-\/>dump\_mmap);}
\DoxyCodeLine{00940\ \ \ \ \ \}\ }
\DoxyCodeLine{00941\ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00942\ \ \ \ \ \ \ \ \ ckd\_free(model-\/>lm3g.bigrams);}
\DoxyCodeLine{00943\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (base-\/>n\ >\ 2)\ \{}
\DoxyCodeLine{00944\ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_free(model-\/>lm3g.trigrams);}
\DoxyCodeLine{00945\ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_free(model-\/>lm3g.tseg\_base);}
\DoxyCodeLine{00946\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00947\ \ \ \ \ \}}
\DoxyCodeLine{00948\ \ \ \ \ \textcolor{keywordflow}{if}\ (base-\/>n\ >\ 2)\ \{}
\DoxyCodeLine{00949\ \ \ \ \ \ \ \ \ ckd\_free(model-\/>lm3g.bo\_wt2);}
\DoxyCodeLine{00950\ \ \ \ \ \ \ \ \ ckd\_free(model-\/>lm3g.prob3);}
\DoxyCodeLine{00951\ \ \ \ \ \}}
\DoxyCodeLine{00952\ }
\DoxyCodeLine{00953\ \ \ \ \ lm3g\_tginfo\_free(base,\ \&model-\/>lm3g);}
\DoxyCodeLine{00954\ \}}
\DoxyCodeLine{00955\ }
\DoxyCodeLine{00956\ \textcolor{keyword}{static}\ ngram\_funcs\_t\ ngram\_model\_dmp\_funcs\ =\ \{}
\DoxyCodeLine{00957\ \ \ \ \ ngram\_model\_dmp\_free,\ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ free\ */}}
\DoxyCodeLine{00958\ \ \ \ \ ngram\_model\_dmp\_apply\_weights,\ \textcolor{comment}{/*\ apply\_weights\ */}}
\DoxyCodeLine{00959\ \ \ \ \ lm3g\_template\_score,\ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ score\ */}}
\DoxyCodeLine{00960\ \ \ \ \ lm3g\_template\_raw\_score,\ \ \ \ \ \ \ \textcolor{comment}{/*\ raw\_score\ */}}
\DoxyCodeLine{00961\ \ \ \ \ lm3g\_template\_add\_ug,\ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ add\_ug\ */}}
\DoxyCodeLine{00962\ \ \ \ \ lm3g\_template\_flush,\ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ flush\ */}}
\DoxyCodeLine{00963\ \ \ \ \ lm3g\_template\_iter,\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ iter\ */}}
\DoxyCodeLine{00964\ \ \ \ \ lm3g\_template\_mgrams,\ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ mgrams\ */}}
\DoxyCodeLine{00965\ \ \ \ \ lm3g\_template\_successors,\ \ \ \ \ \ \textcolor{comment}{/*\ successors\ */}}
\DoxyCodeLine{00966\ \ \ \ \ lm3g\_template\_iter\_get,\ \ \ \ \ \ \ \ \textcolor{comment}{/*\ iter\_get\ */}}
\DoxyCodeLine{00967\ \ \ \ \ lm3g\_template\_iter\_next,\ \ \ \ \ \ \ \textcolor{comment}{/*\ iter\_next\ */}}
\DoxyCodeLine{00968\ \ \ \ \ lm3g\_template\_iter\_free\ \ \ \ \ \ \ \ \textcolor{comment}{/*\ iter\_free\ */}}
\DoxyCodeLine{00969\ \};}

\end{DoxyCode}
