\doxysection{ngram\+\_\+model\+\_\+arpa.\+c}
\label{ngram__model__arpa_8c_source}\index{src/libsphinxbase/lm/ngram\_model\_arpa.c@{src/libsphinxbase/lm/ngram\_model\_arpa.c}}

\begin{DoxyCode}{0}
\DoxyCodeLine{00001\ \textcolor{comment}{/*\ -\/*-\/\ c-\/basic-\/offset:\ 4;\ indent-\/tabs-\/mode:\ nil\ -\/*-\/\ */}}
\DoxyCodeLine{00002\ \textcolor{comment}{/*\ ====================================================================}}
\DoxyCodeLine{00003\ \textcolor{comment}{\ *\ Copyright\ (c)\ 1999-\/2007\ Carnegie\ Mellon\ University.\ \ All\ rights}}
\DoxyCodeLine{00004\ \textcolor{comment}{\ *\ reserved.}}
\DoxyCodeLine{00005\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00006\ \textcolor{comment}{\ *\ Redistribution\ and\ use\ in\ source\ and\ binary\ forms,\ with\ or\ without}}
\DoxyCodeLine{00007\ \textcolor{comment}{\ *\ modification,\ are\ permitted\ provided\ that\ the\ following\ conditions}}
\DoxyCodeLine{00008\ \textcolor{comment}{\ *\ are\ met:}}
\DoxyCodeLine{00009\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00010\ \textcolor{comment}{\ *\ 1.\ Redistributions\ of\ source\ code\ must\ retain\ the\ above\ copyright}}
\DoxyCodeLine{00011\ \textcolor{comment}{\ *\ \ \ \ notice,\ this\ list\ of\ conditions\ and\ the\ following\ disclaimer.\ }}
\DoxyCodeLine{00012\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00013\ \textcolor{comment}{\ *\ 2.\ Redistributions\ in\ binary\ form\ must\ reproduce\ the\ above\ copyright}}
\DoxyCodeLine{00014\ \textcolor{comment}{\ *\ \ \ \ notice,\ this\ list\ of\ conditions\ and\ the\ following\ disclaimer\ in}}
\DoxyCodeLine{00015\ \textcolor{comment}{\ *\ \ \ \ the\ documentation\ and/or\ other\ materials\ provided\ with\ the}}
\DoxyCodeLine{00016\ \textcolor{comment}{\ *\ \ \ \ distribution.}}
\DoxyCodeLine{00017\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00018\ \textcolor{comment}{\ *\ This\ work\ was\ supported\ in\ part\ by\ funding\ from\ the\ Defense\ Advanced\ }}
\DoxyCodeLine{00019\ \textcolor{comment}{\ *\ Research\ Projects\ Agency\ and\ the\ National\ Science\ Foundation\ of\ the\ }}
\DoxyCodeLine{00020\ \textcolor{comment}{\ *\ United\ States\ of\ America,\ and\ the\ CMU\ Sphinx\ Speech\ Consortium.}}
\DoxyCodeLine{00021\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00022\ \textcolor{comment}{\ *\ THIS\ SOFTWARE\ IS\ PROVIDED\ BY\ CARNEGIE\ MELLON\ UNIVERSITY\ \`{}\`{}AS\ IS''\ AND\ }}
\DoxyCodeLine{00023\ \textcolor{comment}{\ *\ ANY\ EXPRESSED\ OR\ IMPLIED\ WARRANTIES,\ INCLUDING,\ BUT\ NOT\ LIMITED\ TO,\ }}
\DoxyCodeLine{00024\ \textcolor{comment}{\ *\ THE\ IMPLIED\ WARRANTIES\ OF\ MERCHANTABILITY\ AND\ FITNESS\ FOR\ A\ PARTICULAR}}
\DoxyCodeLine{00025\ \textcolor{comment}{\ *\ PURPOSE\ ARE\ DISCLAIMED.\ \ IN\ NO\ EVENT\ SHALL\ CARNEGIE\ MELLON\ UNIVERSITY}}
\DoxyCodeLine{00026\ \textcolor{comment}{\ *\ NOR\ ITS\ EMPLOYEES\ BE\ LIABLE\ FOR\ ANY\ DIRECT,\ INDIRECT,\ INCIDENTAL,}}
\DoxyCodeLine{00027\ \textcolor{comment}{\ *\ SPECIAL,\ EXEMPLARY,\ OR\ CONSEQUENTIAL\ DAMAGES\ (INCLUDING,\ BUT\ NOT\ }}
\DoxyCodeLine{00028\ \textcolor{comment}{\ *\ LIMITED\ TO,\ PROCUREMENT\ OF\ SUBSTITUTE\ GOODS\ OR\ SERVICES;\ LOSS\ OF\ USE,\ }}
\DoxyCodeLine{00029\ \textcolor{comment}{\ *\ DATA,\ OR\ PROFITS;\ OR\ BUSINESS\ INTERRUPTION)\ HOWEVER\ CAUSED\ AND\ ON\ ANY\ }}
\DoxyCodeLine{00030\ \textcolor{comment}{\ *\ THEORY\ OF\ LIABILITY,\ WHETHER\ IN\ CONTRACT,\ STRICT\ LIABILITY,\ OR\ TORT\ }}
\DoxyCodeLine{00031\ \textcolor{comment}{\ *\ (INCLUDING\ NEGLIGENCE\ OR\ OTHERWISE)\ ARISING\ IN\ ANY\ WAY\ OUT\ OF\ THE\ USE\ }}
\DoxyCodeLine{00032\ \textcolor{comment}{\ *\ OF\ THIS\ SOFTWARE,\ EVEN\ IF\ ADVISED\ OF\ THE\ POSSIBILITY\ OF\ SUCH\ DAMAGE.}}
\DoxyCodeLine{00033\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00034\ \textcolor{comment}{\ *\ ====================================================================}}
\DoxyCodeLine{00035\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00036\ \textcolor{comment}{\ */}}
\DoxyCodeLine{00037\ \textcolor{comment}{/*}}
\DoxyCodeLine{00038\ \textcolor{comment}{\ *\ \(\backslash\)file\ ngram\_model\_arpa.c\ ARPA\ format\ language\ models}}
\DoxyCodeLine{00039\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00040\ \textcolor{comment}{\ *\ Author:\ David\ Huggins-\/Daines\ <dhuggins@cs.cmu.edu>}}
\DoxyCodeLine{00041\ \textcolor{comment}{\ */}}
\DoxyCodeLine{00042\ }
\DoxyCodeLine{00043\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/ckd\_alloc.h"{}}}
\DoxyCodeLine{00044\ \textcolor{preprocessor}{\#include\ <string.h>}}
\DoxyCodeLine{00045\ \textcolor{preprocessor}{\#include\ <limits.h>}}
\DoxyCodeLine{00046\ \textcolor{preprocessor}{\#include\ <assert.h>}}
\DoxyCodeLine{00047\ }
\DoxyCodeLine{00048\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/err.h"{}}}
\DoxyCodeLine{00049\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/pio.h"{}}}
\DoxyCodeLine{00050\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/listelem\_alloc.h"{}}}
\DoxyCodeLine{00051\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/strfuncs.h"{}}}
\DoxyCodeLine{00052\ }
\DoxyCodeLine{00053\ \textcolor{preprocessor}{\#include\ "{}ngram\_model\_arpa.h"{}}}
\DoxyCodeLine{00054\ }
\DoxyCodeLine{00055\ \textcolor{keyword}{static}\ ngram\_funcs\_t\ ngram\_model\_arpa\_funcs;}
\DoxyCodeLine{00056\ }
\DoxyCodeLine{00057\ \textcolor{preprocessor}{\#define\ TSEG\_BASE(m,b)\ \ \ \ \ \ \ \ \ \ ((m)-\/>lm3g.tseg\_base[(b)>>LOG\_BG\_SEG\_SZ])}}
\DoxyCodeLine{00058\ \textcolor{preprocessor}{\#define\ FIRST\_BG(m,u)\ \ \ \ \ \ \ \ \ \ \ ((m)-\/>lm3g.unigrams[u].bigrams)}}
\DoxyCodeLine{00059\ \textcolor{preprocessor}{\#define\ FIRST\_TG(m,b)\ \ \ \ \ \ \ \ \ \ \ (TSEG\_BASE((m),(b))+((m)-\/>lm3g.bigrams[b].trigrams))}}
\DoxyCodeLine{00060\ }
\DoxyCodeLine{00061\ \textcolor{comment}{/*}}
\DoxyCodeLine{00062\ \textcolor{comment}{\ *\ Read\ and\ return\ \#unigrams,\ \#bigrams,\ \#trigrams\ as\ stated\ in\ input\ file.}}
\DoxyCodeLine{00063\ \textcolor{comment}{\ */}}
\DoxyCodeLine{00064\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{int}}
\DoxyCodeLine{00065\ ReadNgramCounts(lineiter\_t\ **li,\ int32\ *\ n\_ug,\ int32\ *\ n\_bg,\ int32\ *\ n\_tg)}
\DoxyCodeLine{00066\ \{}
\DoxyCodeLine{00067\ \ \ \ \ int32\ ngram,\ ngram\_cnt;}
\DoxyCodeLine{00068\ }
\DoxyCodeLine{00069\ \ \ \ \ \textcolor{comment}{/*\ skip\ file\ until\ past\ the\ '\(\backslash\)data\(\backslash\)'\ marker\ */}}
\DoxyCodeLine{00070\ \ \ \ \ \textcolor{keywordflow}{while}\ (*li)\ \{}
\DoxyCodeLine{00071\ \ \ \ \ \ \ \ \ string\_trim((*li)-\/>buf,\ STRING\_BOTH);}
\DoxyCodeLine{00072\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (strcmp((*li)-\/>buf,\ \textcolor{stringliteral}{"{}\(\backslash\)\(\backslash\)data\(\backslash\)\(\backslash\)"{}})\ ==\ 0)}
\DoxyCodeLine{00073\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00074\ \ \ \ \ \ \ \ \ *li\ =\ lineiter\_next(*li);}
\DoxyCodeLine{00075\ \ \ \ \ \}}
\DoxyCodeLine{00076\ \ \ \ \ \textcolor{keywordflow}{if}\ (*li\ ==\ NULL\ ||\ strcmp((*li)-\/>buf,\ \textcolor{stringliteral}{"{}\(\backslash\)\(\backslash\)data\(\backslash\)\(\backslash\)"{}})\ !=\ 0)\ \{}
\DoxyCodeLine{00077\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}No\ \(\backslash\)\(\backslash\)data\(\backslash\)\(\backslash\)\ mark\ in\ LM\ file\(\backslash\)n"{}});}
\DoxyCodeLine{00078\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00079\ \ \ \ \ \}}
\DoxyCodeLine{00080\ }
\DoxyCodeLine{00081\ \ \ \ \ *n\_ug\ =\ *n\_bg\ =\ *n\_tg\ =\ 0;}
\DoxyCodeLine{00082\ \ \ \ \ \textcolor{keywordflow}{while}\ ((*li\ =\ lineiter\_next(*li)))\ \{}
\DoxyCodeLine{00083\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (sscanf((*li)-\/>buf,\ \textcolor{stringliteral}{"{}ngram\ \%d=\%d"{}},\ \&ngram,\ \&ngram\_cnt)\ !=\ 2)}
\DoxyCodeLine{00084\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00085\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{switch}\ (ngram)\ \{}
\DoxyCodeLine{00086\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{case}\ 1:}
\DoxyCodeLine{00087\ \ \ \ \ \ \ \ \ \ \ \ \ *n\_ug\ =\ ngram\_cnt;}
\DoxyCodeLine{00088\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00089\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{case}\ 2:}
\DoxyCodeLine{00090\ \ \ \ \ \ \ \ \ \ \ \ \ *n\_bg\ =\ ngram\_cnt;}
\DoxyCodeLine{00091\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00092\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{case}\ 3:}
\DoxyCodeLine{00093\ \ \ \ \ \ \ \ \ \ \ \ \ *n\_tg\ =\ ngram\_cnt;}
\DoxyCodeLine{00094\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00095\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{default}:}
\DoxyCodeLine{00096\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Unknown\ ngram\ (\%d)\(\backslash\)n"{}},\ ngram);}
\DoxyCodeLine{00097\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00098\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00099\ \ \ \ \ \}}
\DoxyCodeLine{00100\ \ \ \ \ \textcolor{keywordflow}{if}\ (*li\ ==\ NULL)\ \{}
\DoxyCodeLine{00101\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}EOF\ while\ reading\ ngram\ counts\(\backslash\)n"{}});}
\DoxyCodeLine{00102\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00103\ \ \ \ \ \}}
\DoxyCodeLine{00104\ }
\DoxyCodeLine{00105\ \ \ \ \ \textcolor{comment}{/*\ Position\ iterator\ to\ the\ unigrams\ header\ '\(\backslash\)1-\/grams:\(\backslash\)'\ */}}
\DoxyCodeLine{00106\ \ \ \ \ \textcolor{keywordflow}{while}\ ((*li\ =\ lineiter\_next(*li)))\ \{}
\DoxyCodeLine{00107\ \ \ \ \ \ \ \ \ string\_trim((*li)-\/>buf,\ STRING\_BOTH);}
\DoxyCodeLine{00108\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (strcmp((*li)-\/>buf,\ \textcolor{stringliteral}{"{}\(\backslash\)\(\backslash\)1-\/grams:"{}})\ ==\ 0)}
\DoxyCodeLine{00109\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00110\ \ \ \ \ \}}
\DoxyCodeLine{00111\ \ \ \ \ \textcolor{keywordflow}{if}\ (*li\ ==\ NULL)\ \{}
\DoxyCodeLine{00112\ \ \ \ \ \ \ \ \ E\_ERROR\_SYSTEM(\textcolor{stringliteral}{"{}Failed\ to\ read\ \(\backslash\)\(\backslash\)1-\/grams:\ mark"{}});}
\DoxyCodeLine{00113\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00114\ \ \ \ \ \}}
\DoxyCodeLine{00115\ }
\DoxyCodeLine{00116\ \ \ \ \ \textcolor{keywordflow}{if}\ ((*n\_ug\ <=\ 0)\ ||\ (*n\_bg\ <\ 0)\ ||\ (*n\_tg\ <\ 0))\ \{}
\DoxyCodeLine{00117\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Bad\ or\ missing\ ngram\ count\(\backslash\)n"{}});}
\DoxyCodeLine{00118\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00119\ \ \ \ \ \}}
\DoxyCodeLine{00120\ \ \ \ \ \textcolor{keywordflow}{return}\ 0;}
\DoxyCodeLine{00121\ \}}
\DoxyCodeLine{00122\ }
\DoxyCodeLine{00123\ \textcolor{comment}{/*}}
\DoxyCodeLine{00124\ \textcolor{comment}{\ *\ Read\ in\ the\ unigrams\ from\ given\ file\ into\ the\ LM\ structure\ model.}}
\DoxyCodeLine{00125\ \textcolor{comment}{\ *\ On\ entry\ to\ this\ procedure,\ the\ iterator\ is\ positioned\ to\ the}}
\DoxyCodeLine{00126\ \textcolor{comment}{\ *\ header\ line\ '\(\backslash\)1-\/grams:'.}}
\DoxyCodeLine{00127\ \textcolor{comment}{\ */}}
\DoxyCodeLine{00128\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{int}}
\DoxyCodeLine{00129\ ReadUnigrams(lineiter\_t\ **li,\ ngram\_model\_arpa\_t\ *\ model)}
\DoxyCodeLine{00130\ \{}
\DoxyCodeLine{00131\ \ \ \ \ ngram\_model\_t\ *base\ =\ \&model-\/>base;}
\DoxyCodeLine{00132\ \ \ \ \ int32\ wcnt;}
\DoxyCodeLine{00133\ \ \ \ \ \textcolor{keywordtype}{float}\ p1;}
\DoxyCodeLine{00134\ }
\DoxyCodeLine{00135\ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}Reading\ unigrams\(\backslash\)n"{}});}
\DoxyCodeLine{00136\ }
\DoxyCodeLine{00137\ \ \ \ \ wcnt\ =\ 0;}
\DoxyCodeLine{00138\ \ \ \ \ \textcolor{keywordflow}{while}\ ((*li\ =\ lineiter\_next(*li)))\ \{}
\DoxyCodeLine{00139\ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{char}\ *wptr[3],\ *name;}
\DoxyCodeLine{00140\ \ \ \ \ \ \ \ \ float32\ bo\_wt\ =\ 0.0f;}
\DoxyCodeLine{00141\ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{int}\ n;}
\DoxyCodeLine{00142\ }
\DoxyCodeLine{00143\ \ \ \ \ \ \ \ \ string\_trim((*li)-\/>buf,\ STRING\_BOTH);}
\DoxyCodeLine{00144\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (strcmp((*li)-\/>buf,\ \textcolor{stringliteral}{"{}\(\backslash\)\(\backslash\)2-\/grams:"{}})\ ==\ 0}
\DoxyCodeLine{00145\ \ \ \ \ \ \ \ \ \ \ \ \ ||\ strcmp((*li)-\/>buf,\ \textcolor{stringliteral}{"{}\(\backslash\)\(\backslash\)end\(\backslash\)\(\backslash\)"{}})\ ==\ 0)}
\DoxyCodeLine{00146\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00147\ }
\DoxyCodeLine{00148\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((n\ =\ str2words((*li)-\/>buf,\ wptr,\ 3))\ <\ 2)\ \{}
\DoxyCodeLine{00149\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((*li)-\/>buf[0]\ !=\ \textcolor{charliteral}{'\(\backslash\)0'})}
\DoxyCodeLine{00150\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_WARN(\textcolor{stringliteral}{"{}Format\ error;\ unigram\ ignored:\ \%s\(\backslash\)n"{}},\ (*li)-\/>buf);}
\DoxyCodeLine{00151\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{continue};}
\DoxyCodeLine{00152\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00153\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00154\ \ \ \ \ \ \ \ \ \ \ \ \ p1\ =\ (float)atof\_c(wptr[0]);}
\DoxyCodeLine{00155\ \ \ \ \ \ \ \ \ \ \ \ \ name\ =\ wptr[1];}
\DoxyCodeLine{00156\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (n\ ==\ 3)}
\DoxyCodeLine{00157\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ bo\_wt\ =\ (float)atof\_c(wptr[2]);}
\DoxyCodeLine{00158\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00159\ }
\DoxyCodeLine{00160\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (wcnt\ >=\ base-\/>n\_counts[0])\ \{}
\DoxyCodeLine{00161\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Too\ many\ unigrams\(\backslash\)n"{}});}
\DoxyCodeLine{00162\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00163\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00164\ }
\DoxyCodeLine{00165\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Associate\ name\ with\ word\ id\ */}}
\DoxyCodeLine{00166\ \ \ \ \ \ \ \ \ base-\/>word\_str[wcnt]\ =\ ckd\_salloc(name);}
\DoxyCodeLine{00167\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((hash\_table\_enter(base-\/>wid,\ base-\/>word\_str[wcnt],\ (\textcolor{keywordtype}{void}\ *)(\textcolor{keywordtype}{long})wcnt))}
\DoxyCodeLine{00168\ \ \ \ \ \ \ \ \ \ \ \ \ !=\ (\textcolor{keywordtype}{void}\ *)(\textcolor{keywordtype}{long})wcnt)\ \{}
\DoxyCodeLine{00169\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_WARN(\textcolor{stringliteral}{"{}Duplicate\ word\ in\ dictionary:\ \%s\(\backslash\)n"{}},\ base-\/>word\_str[wcnt]);}
\DoxyCodeLine{00170\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00171\ \ \ \ \ \ \ \ \ model-\/>lm3g.unigrams[wcnt].prob1.l\ =\ logmath\_log10\_to\_log(base-\/>lmath,\ p1);}
\DoxyCodeLine{00172\ \ \ \ \ \ \ \ \ model-\/>lm3g.unigrams[wcnt].bo\_wt1.l\ =\ logmath\_log10\_to\_log(base-\/>lmath,\ bo\_wt);}
\DoxyCodeLine{00173\ \ \ \ \ \ \ \ \ wcnt++;}
\DoxyCodeLine{00174\ \ \ \ \ \}}
\DoxyCodeLine{00175\ }
\DoxyCodeLine{00176\ \ \ \ \ \textcolor{keywordflow}{if}\ (base-\/>n\_counts[0]\ !=\ wcnt)\ \{}
\DoxyCodeLine{00177\ \ \ \ \ \ \ \ \ E\_WARN(\textcolor{stringliteral}{"{}lm\_t.ucount(\%d)\ !=\ \#unigrams\ read(\%d)\(\backslash\)n"{}},}
\DoxyCodeLine{00178\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ base-\/>n\_counts[0],\ wcnt);}
\DoxyCodeLine{00179\ \ \ \ \ \ \ \ \ base-\/>n\_counts[0]\ =\ wcnt;}
\DoxyCodeLine{00180\ \ \ \ \ \ \ \ \ base-\/>n\_words\ =\ wcnt;}
\DoxyCodeLine{00181\ \ \ \ \ \}}
\DoxyCodeLine{00182\ \ \ \ \ \textcolor{keywordflow}{return}\ 0;}
\DoxyCodeLine{00183\ \}}
\DoxyCodeLine{00184\ }
\DoxyCodeLine{00185\ \textcolor{comment}{/*}}
\DoxyCodeLine{00186\ \textcolor{comment}{\ *\ Read\ bigrams\ from\ given\ file\ into\ given\ model\ structure.}}
\DoxyCodeLine{00187\ \textcolor{comment}{\ */}}
\DoxyCodeLine{00188\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{int}}
\DoxyCodeLine{00189\ ReadBigrams(lineiter\_t\ **li,\ ngram\_model\_arpa\_t\ *\ model)}
\DoxyCodeLine{00190\ \{}
\DoxyCodeLine{00191\ \ \ \ \ ngram\_model\_t\ *base\ =\ \&model-\/>base;}
\DoxyCodeLine{00192\ \ \ \ \ int32\ w1,\ w2,\ prev\_w1,\ bgcount;}
\DoxyCodeLine{00193\ \ \ \ \ bigram\_t\ *bgptr;}
\DoxyCodeLine{00194\ }
\DoxyCodeLine{00195\ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}Reading\ bigrams\(\backslash\)n"{}});}
\DoxyCodeLine{00196\ }
\DoxyCodeLine{00197\ \ \ \ \ bgcount\ =\ 0;}
\DoxyCodeLine{00198\ \ \ \ \ bgptr\ =\ model-\/>lm3g.bigrams;}
\DoxyCodeLine{00199\ \ \ \ \ prev\_w1\ =\ -\/1;}
\DoxyCodeLine{00200\ }
\DoxyCodeLine{00201\ \ \ \ \ \textcolor{keywordflow}{while}\ ((*li\ =\ lineiter\_next(*li)))\ \{}
\DoxyCodeLine{00202\ \ \ \ \ \ \ \ \ float32\ p,\ bo\_wt\ =\ 0.0f;}
\DoxyCodeLine{00203\ \ \ \ \ \ \ \ \ int32\ p2,\ bo\_wt2;}
\DoxyCodeLine{00204\ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{char}\ *wptr[4],\ *word1,\ *word2;}
\DoxyCodeLine{00205\ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{int}\ n;}
\DoxyCodeLine{00206\ }
\DoxyCodeLine{00207\ \ \ \ \ \ \ \ \ string\_trim((*li)-\/>buf,\ STRING\_BOTH);}
\DoxyCodeLine{00208\ \ \ \ \ \ \ \ \ wptr[3]\ =\ NULL;}
\DoxyCodeLine{00209\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((n\ =\ str2words((*li)-\/>buf,\ wptr,\ 4))\ <\ 3)\ \{}
\DoxyCodeLine{00210\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((*li)-\/>buf[0]\ !=\ \textcolor{charliteral}{'\(\backslash\)0'})}
\DoxyCodeLine{00211\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00212\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{continue};}
\DoxyCodeLine{00213\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00214\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00215\ \ \ \ \ \ \ \ \ \ \ \ \ p\ =\ (float32)atof\_c(wptr[0]);}
\DoxyCodeLine{00216\ \ \ \ \ \ \ \ \ \ \ \ \ word1\ =\ wptr[1];}
\DoxyCodeLine{00217\ \ \ \ \ \ \ \ \ \ \ \ \ word2\ =\ wptr[2];}
\DoxyCodeLine{00218\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (wptr[3])}
\DoxyCodeLine{00219\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ bo\_wt\ =\ (float32)atof\_c(wptr[3]);}
\DoxyCodeLine{00220\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00221\ }
\DoxyCodeLine{00222\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((w1\ =\ ngram\_wid(base,\ word1))\ ==\ NGRAM\_INVALID\_WID)\ \{}
\DoxyCodeLine{00223\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Unknown\ word:\ \%s,\ skipping\ bigram\ (\%s\ \%s)\(\backslash\)n"{}},}
\DoxyCodeLine{00224\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ word1,\ word1,\ word2);}
\DoxyCodeLine{00225\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{continue};}
\DoxyCodeLine{00226\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00227\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((w2\ =\ ngram\_wid(base,\ word2))\ ==\ NGRAM\_INVALID\_WID)\ \{}
\DoxyCodeLine{00228\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Unknown\ word:\ \%s,\ skipping\ bigram\ (\%s\ \%s)\(\backslash\)n"{}},}
\DoxyCodeLine{00229\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ word2,\ word1,\ word2);}
\DoxyCodeLine{00230\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{continue};}
\DoxyCodeLine{00231\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00232\ }
\DoxyCodeLine{00233\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ FIXME:\ Should\ use\ logmath\_t\ quantization\ here.\ */}}
\DoxyCodeLine{00234\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ HACK!!\ to\ quantize\ probs\ to\ 4\ decimal\ digits\ */}}
\DoxyCodeLine{00235\ \ \ \ \ \ \ \ \ p\ =\ (float32)((int32)(p\ *\ 10000))\ /\ 10000;}
\DoxyCodeLine{00236\ \ \ \ \ \ \ \ \ bo\_wt\ =\ (float32)((int32)(bo\_wt\ *\ 10000))\ /\ 10000;}
\DoxyCodeLine{00237\ }
\DoxyCodeLine{00238\ \ \ \ \ \ \ \ \ p2\ =\ logmath\_log10\_to\_log(base-\/>lmath,\ p);}
\DoxyCodeLine{00239\ \ \ \ \ \ \ \ \ bo\_wt2\ =\ logmath\_log10\_to\_log(base-\/>lmath,\ bo\_wt);}
\DoxyCodeLine{00240\ }
\DoxyCodeLine{00241\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (bgcount\ >=\ base-\/>n\_counts[1])\ \{}
\DoxyCodeLine{00242\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Too\ many\ bigrams\(\backslash\)n"{}});}
\DoxyCodeLine{00243\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00244\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00245\ }
\DoxyCodeLine{00246\ \ \ \ \ \ \ \ \ bgptr-\/>wid\ =\ w2;}
\DoxyCodeLine{00247\ \ \ \ \ \ \ \ \ bgptr-\/>prob2\ =\ sorted\_id(\&model-\/>sorted\_prob2,\ \&p2);}
\DoxyCodeLine{00248\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (base-\/>n\_counts[2]\ >\ 0)}
\DoxyCodeLine{00249\ \ \ \ \ \ \ \ \ \ \ \ \ bgptr-\/>bo\_wt2\ =\ sorted\_id(\&model-\/>sorted\_bo\_wt2,\ \&bo\_wt2);}
\DoxyCodeLine{00250\ }
\DoxyCodeLine{00251\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (w1\ !=\ prev\_w1)\ \{}
\DoxyCodeLine{00252\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (w1\ <\ prev\_w1)\ \{}
\DoxyCodeLine{00253\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Bigrams\ not\ in\ unigram\ order\(\backslash\)n"{}});}
\DoxyCodeLine{00254\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00255\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00256\ }
\DoxyCodeLine{00257\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (prev\_w1++;\ prev\_w1\ <=\ w1;\ prev\_w1++)}
\DoxyCodeLine{00258\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>lm3g.unigrams[prev\_w1].bigrams\ =\ bgcount;}
\DoxyCodeLine{00259\ \ \ \ \ \ \ \ \ \ \ \ \ prev\_w1\ =\ w1;}
\DoxyCodeLine{00260\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00261\ \ \ \ \ \ \ \ \ bgcount++;}
\DoxyCodeLine{00262\ \ \ \ \ \ \ \ \ bgptr++;}
\DoxyCodeLine{00263\ }
\DoxyCodeLine{00264\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((bgcount\ \&\ 0x0000ffff)\ ==\ 0)\ \{}
\DoxyCodeLine{00265\ \ \ \ \ \ \ \ \ \ \ \ \ E\_INFOCONT(\textcolor{stringliteral}{"{}."{}});}
\DoxyCodeLine{00266\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00267\ \ \ \ \ \}}
\DoxyCodeLine{00268\ \ \ \ \ \textcolor{keywordflow}{if}\ (*li\ ==\ NULL\ ||\ ((strcmp((*li)-\/>buf,\ \textcolor{stringliteral}{"{}\(\backslash\)\(\backslash\)end\(\backslash\)\(\backslash\)"{}})\ !=\ 0)}
\DoxyCodeLine{00269\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \&\&\ (strcmp((*li)-\/>buf,\ \textcolor{stringliteral}{"{}\(\backslash\)\(\backslash\)3-\/grams:"{}})\ !=\ 0)))\ \{}
\DoxyCodeLine{00270\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Bad\ bigram:\ \%s\(\backslash\)n"{}},\ (*li)-\/>buf);}
\DoxyCodeLine{00271\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00272\ \ \ \ \ \}}
\DoxyCodeLine{00273\ }
\DoxyCodeLine{00274\ \ \ \ \ \textcolor{keywordflow}{for}\ (prev\_w1++;\ prev\_w1\ <=\ base-\/>n\_counts[0];\ prev\_w1++)}
\DoxyCodeLine{00275\ \ \ \ \ \ \ \ \ model-\/>lm3g.unigrams[prev\_w1].bigrams\ =\ bgcount;}
\DoxyCodeLine{00276\ }
\DoxyCodeLine{00277\ \ \ \ \ \textcolor{keywordflow}{return}\ 0;}
\DoxyCodeLine{00278\ \}}
\DoxyCodeLine{00279\ }
\DoxyCodeLine{00280\ \textcolor{comment}{/*}}
\DoxyCodeLine{00281\ \textcolor{comment}{\ *\ Very\ similar\ to\ ReadBigrams.}}
\DoxyCodeLine{00282\ \textcolor{comment}{\ */}}
\DoxyCodeLine{00283\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{int}}
\DoxyCodeLine{00284\ ReadTrigrams(lineiter\_t\ **li,\ ngram\_model\_arpa\_t\ *\ model)}
\DoxyCodeLine{00285\ \{}
\DoxyCodeLine{00286\ \ \ \ \ ngram\_model\_t\ *base\ =\ \&model-\/>base;}
\DoxyCodeLine{00287\ \ \ \ \ int32\ i,\ w1,\ w2,\ w3,\ prev\_w1,\ prev\_w2,\ tgcount,\ prev\_bg,\ bg,\ endbg;}
\DoxyCodeLine{00288\ \ \ \ \ int32\ seg,\ prev\_seg,\ prev\_seg\_lastbg;}
\DoxyCodeLine{00289\ \ \ \ \ trigram\_t\ *tgptr;}
\DoxyCodeLine{00290\ \ \ \ \ bigram\_t\ *bgptr;}
\DoxyCodeLine{00291\ }
\DoxyCodeLine{00292\ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}Reading\ trigrams\(\backslash\)n"{}});}
\DoxyCodeLine{00293\ }
\DoxyCodeLine{00294\ \ \ \ \ tgcount\ =\ 0;}
\DoxyCodeLine{00295\ \ \ \ \ tgptr\ =\ model-\/>lm3g.trigrams;}
\DoxyCodeLine{00296\ \ \ \ \ prev\_w1\ =\ -\/1;}
\DoxyCodeLine{00297\ \ \ \ \ prev\_w2\ =\ -\/1;}
\DoxyCodeLine{00298\ \ \ \ \ prev\_bg\ =\ -\/1;}
\DoxyCodeLine{00299\ \ \ \ \ prev\_seg\ =\ -\/1;}
\DoxyCodeLine{00300\ }
\DoxyCodeLine{00301\ \ \ \ \ \textcolor{keywordflow}{while}\ ((*li\ =\ lineiter\_next(*li)))\ \{}
\DoxyCodeLine{00302\ \ \ \ \ \ \ \ \ float32\ p;}
\DoxyCodeLine{00303\ \ \ \ \ \ \ \ \ int32\ p3;}
\DoxyCodeLine{00304\ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{char}\ *wptr[4],\ *word1,\ *word2,\ *word3;}
\DoxyCodeLine{00305\ }
\DoxyCodeLine{00306\ \ \ \ \ \ \ \ \ string\_trim((*li)-\/>buf,\ STRING\_BOTH);}
\DoxyCodeLine{00307\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (str2words((*li)-\/>buf,\ wptr,\ 4)\ !=\ 4)\ \{}
\DoxyCodeLine{00308\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((*li)-\/>buf[0]\ !=\ \textcolor{charliteral}{'\(\backslash\)0'})}
\DoxyCodeLine{00309\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00310\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{continue};}
\DoxyCodeLine{00311\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00312\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00313\ \ \ \ \ \ \ \ \ \ \ \ \ p\ =\ (float32)atof\_c(wptr[0]);}
\DoxyCodeLine{00314\ \ \ \ \ \ \ \ \ \ \ \ \ word1\ =\ wptr[1];}
\DoxyCodeLine{00315\ \ \ \ \ \ \ \ \ \ \ \ \ word2\ =\ wptr[2];}
\DoxyCodeLine{00316\ \ \ \ \ \ \ \ \ \ \ \ \ word3\ =\ wptr[3];}
\DoxyCodeLine{00317\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00318\ }
\DoxyCodeLine{00319\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((w1\ =\ ngram\_wid(base,\ word1))\ ==\ NGRAM\_INVALID\_WID)\ \{}
\DoxyCodeLine{00320\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Unknown\ word:\ \%s,\ skipping\ trigram\ (\%s\ \%s\ \%s)\(\backslash\)n"{}},}
\DoxyCodeLine{00321\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ word1,\ word1,\ word2,\ word3);}
\DoxyCodeLine{00322\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{continue};}
\DoxyCodeLine{00323\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00324\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((w2\ =\ ngram\_wid(base,\ word2))\ ==\ NGRAM\_INVALID\_WID)\ \{}
\DoxyCodeLine{00325\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Unknown\ word:\ \%s,\ skipping\ trigram\ (\%s\ \%s\ \%s)\(\backslash\)n"{}},}
\DoxyCodeLine{00326\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ word2,\ word1,\ word2,\ word3);}
\DoxyCodeLine{00327\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{continue};}
\DoxyCodeLine{00328\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00329\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((w3\ =\ ngram\_wid(base,\ word3))\ ==\ NGRAM\_INVALID\_WID)\ \{}
\DoxyCodeLine{00330\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Unknown\ word:\ \%s,\ skipping\ trigram\ (\%s\ \%s\ \%s)\(\backslash\)n"{}},}
\DoxyCodeLine{00331\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ word3,\ word1,\ word2,\ word3);}
\DoxyCodeLine{00332\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{continue};}
\DoxyCodeLine{00333\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00334\ }
\DoxyCodeLine{00335\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ FIXME:\ Should\ use\ logmath\_t\ quantization\ here.\ */}}
\DoxyCodeLine{00336\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ HACK!!\ to\ quantize\ probs\ to\ 4\ decimal\ digits\ */}}
\DoxyCodeLine{00337\ \ \ \ \ \ \ \ \ p\ =\ (float32)((int32)(p\ *\ 10000))\ /\ 10000;}
\DoxyCodeLine{00338\ \ \ \ \ \ \ \ \ p3\ =\ logmath\_log10\_to\_log(base-\/>lmath,\ p);}
\DoxyCodeLine{00339\ }
\DoxyCodeLine{00340\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (tgcount\ >=\ base-\/>n\_counts[2])\ \{}
\DoxyCodeLine{00341\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Too\ many\ trigrams\(\backslash\)n"{}});}
\DoxyCodeLine{00342\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00343\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00344\ }
\DoxyCodeLine{00345\ \ \ \ \ \ \ \ \ tgptr-\/>wid\ =\ w3;}
\DoxyCodeLine{00346\ \ \ \ \ \ \ \ \ tgptr-\/>prob3\ =\ sorted\_id(\&model-\/>sorted\_prob3,\ \&p3);}
\DoxyCodeLine{00347\ }
\DoxyCodeLine{00348\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((w1\ !=\ prev\_w1)\ ||\ (w2\ !=\ prev\_w2))\ \{}
\DoxyCodeLine{00349\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Trigram\ for\ a\ new\ bigram;\ update\ tg\ info\ for\ all\ previous\ bigrams\ */}}
\DoxyCodeLine{00350\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((w1\ <\ prev\_w1)\ ||\ ((w1\ ==\ prev\_w1)\ \&\&\ (w2\ <\ prev\_w2)))\ \{}
\DoxyCodeLine{00351\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Trigrams\ not\ in\ bigram\ order\(\backslash\)n"{}});}
\DoxyCodeLine{00352\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00353\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00354\ }
\DoxyCodeLine{00355\ \ \ \ \ \ \ \ \ \ \ \ \ bg\ =\ (w1\ !=}
\DoxyCodeLine{00356\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ prev\_w1)\ ?\ model-\/>lm3g.unigrams[w1].bigrams\ :\ prev\_bg\ +\ 1;}
\DoxyCodeLine{00357\ \ \ \ \ \ \ \ \ \ \ \ \ endbg\ =\ model-\/>lm3g.unigrams[w1\ +\ 1].bigrams;}
\DoxyCodeLine{00358\ \ \ \ \ \ \ \ \ \ \ \ \ bgptr\ =\ model-\/>lm3g.bigrams\ +\ bg;}
\DoxyCodeLine{00359\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (;\ (bg\ <\ endbg)\ \&\&\ (bgptr-\/>wid\ !=\ w2);\ bg++,\ bgptr++);}
\DoxyCodeLine{00360\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (bg\ >=\ endbg)\ \{}
\DoxyCodeLine{00361\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Missing\ bigram\ for\ trigram:\ \%s"{}},\ (*li)-\/>buf);}
\DoxyCodeLine{00362\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00363\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00364\ }
\DoxyCodeLine{00365\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ bg\ =\ bigram\ entry\ index\ for\ <w1,w2>.\ \ Update\ tseg\_base\ */}}
\DoxyCodeLine{00366\ \ \ \ \ \ \ \ \ \ \ \ \ seg\ =\ bg\ >>\ LOG\_BG\_SEG\_SZ;}
\DoxyCodeLine{00367\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ prev\_seg\ +\ 1;\ i\ <=\ seg;\ i++)}
\DoxyCodeLine{00368\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>lm3g.tseg\_base[i]\ =\ tgcount;}
\DoxyCodeLine{00369\ }
\DoxyCodeLine{00370\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Update\ trigrams\ pointers\ for\ all\ bigrams\ until\ bg\ */}}
\DoxyCodeLine{00371\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (prev\_seg\ <\ seg)\ \{}
\DoxyCodeLine{00372\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ tgoff\ =\ 0;}
\DoxyCodeLine{00373\ }
\DoxyCodeLine{00374\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (prev\_seg\ >=\ 0)\ \{}
\DoxyCodeLine{00375\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ tgoff\ =\ tgcount\ -\/\ model-\/>lm3g.tseg\_base[prev\_seg];}
\DoxyCodeLine{00376\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (tgoff\ >\ 65535)\ \{}
\DoxyCodeLine{00377\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Size\ of\ trigram\ segment\ is\ bigger\ than\ 65535,\ such\ a\ big\ language\ models\ are\ not\ supported,\ use\ smaller\ vocabulary\(\backslash\)n"{}});}
\DoxyCodeLine{00378\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00379\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00380\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00381\ }
\DoxyCodeLine{00382\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ prev\_seg\_lastbg\ =\ ((prev\_seg\ +\ 1)\ <<\ LOG\_BG\_SEG\_SZ)\ -\/\ 1;}
\DoxyCodeLine{00383\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ bgptr\ =\ model-\/>lm3g.bigrams\ +\ prev\_bg;}
\DoxyCodeLine{00384\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (++prev\_bg,\ ++bgptr;\ prev\_bg\ <=\ prev\_seg\_lastbg;}
\DoxyCodeLine{00385\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ prev\_bg++,\ bgptr++)}
\DoxyCodeLine{00386\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ bgptr-\/>trigrams\ =\ tgoff;}
\DoxyCodeLine{00387\ }
\DoxyCodeLine{00388\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (;\ prev\_bg\ <=\ bg;\ prev\_bg++,\ bgptr++)}
\DoxyCodeLine{00389\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ bgptr-\/>trigrams\ =\ 0;}
\DoxyCodeLine{00390\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00391\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00392\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ tgoff;}
\DoxyCodeLine{00393\ }
\DoxyCodeLine{00394\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ tgoff\ =\ tgcount\ -\/\ model-\/>lm3g.tseg\_base[prev\_seg];}
\DoxyCodeLine{00395\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (tgoff\ >\ 65535)\ \{}
\DoxyCodeLine{00396\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Size\ of\ trigram\ segment\ is\ bigger\ than\ 65535,\ such\ a\ big\ language\ models\ are\ not\ supported,\ use\ smaller\ vocabulary\(\backslash\)n"{}});}
\DoxyCodeLine{00397\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00398\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00399\ }
\DoxyCodeLine{00400\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ bgptr\ =\ model-\/>lm3g.bigrams\ +\ prev\_bg;}
\DoxyCodeLine{00401\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (++prev\_bg,\ ++bgptr;\ prev\_bg\ <=\ bg;\ prev\_bg++,\ bgptr++)}
\DoxyCodeLine{00402\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ bgptr-\/>trigrams\ =\ tgoff;}
\DoxyCodeLine{00403\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00404\ }
\DoxyCodeLine{00405\ \ \ \ \ \ \ \ \ \ \ \ \ prev\_w1\ =\ w1;}
\DoxyCodeLine{00406\ \ \ \ \ \ \ \ \ \ \ \ \ prev\_w2\ =\ w2;}
\DoxyCodeLine{00407\ \ \ \ \ \ \ \ \ \ \ \ \ prev\_bg\ =\ bg;}
\DoxyCodeLine{00408\ \ \ \ \ \ \ \ \ \ \ \ \ prev\_seg\ =\ seg;}
\DoxyCodeLine{00409\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00410\ }
\DoxyCodeLine{00411\ \ \ \ \ \ \ \ \ tgcount++;}
\DoxyCodeLine{00412\ \ \ \ \ \ \ \ \ tgptr++;}
\DoxyCodeLine{00413\ }
\DoxyCodeLine{00414\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((tgcount\ \&\ 0x0000ffff)\ ==\ 0)\ \{}
\DoxyCodeLine{00415\ \ \ \ \ \ \ \ \ \ \ \ \ E\_INFOCONT(\textcolor{stringliteral}{"{}."{}});}
\DoxyCodeLine{00416\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00417\ \ \ \ \ \}}
\DoxyCodeLine{00418\ \ \ \ \ \textcolor{keywordflow}{if}\ (*li\ ==\ NULL\ ||\ strcmp((*li)-\/>buf,\ \textcolor{stringliteral}{"{}\(\backslash\)\(\backslash\)end\(\backslash\)\(\backslash\)"{}})\ !=\ 0)\ \{}
\DoxyCodeLine{00419\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Bad\ trigram:\ \%s\(\backslash\)n"{}},\ (*li)-\/>buf);}
\DoxyCodeLine{00420\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00421\ \ \ \ \ \}}
\DoxyCodeLine{00422\ }
\DoxyCodeLine{00423\ \ \ \ \ \textcolor{keywordflow}{for}\ (prev\_bg++;\ prev\_bg\ <=\ base-\/>n\_counts[1];\ prev\_bg++)\ \{}
\DoxyCodeLine{00424\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((prev\_bg\ \&\ (BG\_SEG\_SZ\ -\/\ 1))\ ==\ 0)}
\DoxyCodeLine{00425\ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>lm3g.tseg\_base[prev\_bg\ >>\ LOG\_BG\_SEG\_SZ]\ =\ tgcount;}
\DoxyCodeLine{00426\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((tgcount\ -\/\ model-\/>lm3g.tseg\_base[prev\_bg\ >>\ LOG\_BG\_SEG\_SZ])\ >\ 65535)\ \{}
\DoxyCodeLine{00427\ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Size\ of\ trigram\ segment\ is\ bigger\ than\ 65535,\ such\ a\ big\ language\ models\ are\ not\ supported,\ use\ smaller\ vocabulary\(\backslash\)n"{}});}
\DoxyCodeLine{00428\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00429\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00430\ \ \ \ \ \ \ \ \ model-\/>lm3g.bigrams[prev\_bg].trigrams\ =}
\DoxyCodeLine{00431\ \ \ \ \ \ \ \ \ \ \ \ \ tgcount\ -\/\ model-\/>lm3g.tseg\_base[prev\_bg\ >>\ LOG\_BG\_SEG\_SZ];}
\DoxyCodeLine{00432\ \ \ \ \ \}}
\DoxyCodeLine{00433\ \ \ \ \ \textcolor{keywordflow}{return}\ 0;}
\DoxyCodeLine{00434\ \}}
\DoxyCodeLine{00435\ }
\DoxyCodeLine{00436\ \textcolor{keyword}{static}\ unigram\_t\ *}
\DoxyCodeLine{00437\ new\_unigram\_table(int32\ n\_ug)}
\DoxyCodeLine{00438\ \{}
\DoxyCodeLine{00439\ \ \ \ \ unigram\_t\ *table;}
\DoxyCodeLine{00440\ \ \ \ \ int32\ i;}
\DoxyCodeLine{00441\ }
\DoxyCodeLine{00442\ \ \ \ \ table\ =\ ckd\_calloc(n\_ug,\ \textcolor{keyword}{sizeof}(unigram\_t));}
\DoxyCodeLine{00443\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ n\_ug;\ i++)\ \{}
\DoxyCodeLine{00444\ \ \ \ \ \ \ \ \ table[i].prob1.l\ =\ INT\_MIN;}
\DoxyCodeLine{00445\ \ \ \ \ \ \ \ \ table[i].bo\_wt1.l\ =\ INT\_MIN;}
\DoxyCodeLine{00446\ \ \ \ \ \}}
\DoxyCodeLine{00447\ \ \ \ \ \textcolor{keywordflow}{return}\ table;}
\DoxyCodeLine{00448\ \}}
\DoxyCodeLine{00449\ }
\DoxyCodeLine{00450\ ngram\_model\_t\ *}
\DoxyCodeLine{00451\ ngram\_model\_arpa\_read(cmd\_ln\_t\ *config,}
\DoxyCodeLine{00452\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *file\_name,}
\DoxyCodeLine{00453\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ logmath\_t\ *lmath)}
\DoxyCodeLine{00454\ \{}
\DoxyCodeLine{00455\ \ \ \ \ lineiter\_t\ *li;}
\DoxyCodeLine{00456\ \ \ \ \ FILE\ *fp;}
\DoxyCodeLine{00457\ \ \ \ \ int32\ is\_pipe;}
\DoxyCodeLine{00458\ \ \ \ \ int32\ n\_unigram;}
\DoxyCodeLine{00459\ \ \ \ \ int32\ n\_bigram;}
\DoxyCodeLine{00460\ \ \ \ \ int32\ n\_trigram;}
\DoxyCodeLine{00461\ \ \ \ \ int32\ n;}
\DoxyCodeLine{00462\ \ \ \ \ ngram\_model\_arpa\_t\ *model;}
\DoxyCodeLine{00463\ \ \ \ \ ngram\_model\_t\ *base;}
\DoxyCodeLine{00464\ }
\DoxyCodeLine{00465\ \ \ \ \ \textcolor{keywordflow}{if}\ ((fp\ =\ fopen\_comp(file\_name,\ \textcolor{stringliteral}{"{}r"{}},\ \&is\_pipe))\ ==\ NULL)\ \{}
\DoxyCodeLine{00466\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}File\ \%s\ not\ found\(\backslash\)n"{}},\ file\_name);}
\DoxyCodeLine{00467\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00468\ \ \ \ \ \}}
\DoxyCodeLine{00469\ \ \ \ \ li\ =\ lineiter\_start(fp);}
\DoxyCodeLine{00470\ \ }
\DoxyCodeLine{00471\ \ \ \ \ \textcolor{comment}{/*\ Read\ \#unigrams,\ \#bigrams,\ \#trigrams\ from\ file\ */}}
\DoxyCodeLine{00472\ \ \ \ \ \textcolor{keywordflow}{if}\ (ReadNgramCounts(\&li,\ \&n\_unigram,\ \&n\_bigram,\ \&n\_trigram)\ ==\ -\/1)\ \{}
\DoxyCodeLine{00473\ \ \ \ \ \ \ \ \ lineiter\_free(li);}
\DoxyCodeLine{00474\ \ \ \ \ \ \ \ \ fclose\_comp(fp,\ is\_pipe);}
\DoxyCodeLine{00475\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00476\ \ \ \ \ \}}
\DoxyCodeLine{00477\ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}ngrams\ 1=\%d,\ 2=\%d,\ 3=\%d\(\backslash\)n"{}},\ n\_unigram,\ n\_bigram,\ n\_trigram);}
\DoxyCodeLine{00478\ }
\DoxyCodeLine{00479\ \ \ \ \ \textcolor{comment}{/*\ Allocate\ space\ for\ LM,\ including\ initial\ OOVs\ and\ placeholders;\ initialize\ it\ */}}
\DoxyCodeLine{00480\ \ \ \ \ model\ =\ ckd\_calloc(1,\ \textcolor{keyword}{sizeof}(*model));}
\DoxyCodeLine{00481\ \ \ \ \ base\ =\ \&model-\/>base;}
\DoxyCodeLine{00482\ \ \ \ \ \textcolor{keywordflow}{if}\ (n\_trigram\ >\ 0)}
\DoxyCodeLine{00483\ \ \ \ \ \ \ \ \ n\ =\ 3;}
\DoxyCodeLine{00484\ \ \ \ \ \textcolor{keywordflow}{else}\ \textcolor{keywordflow}{if}\ (n\_bigram\ >\ 0)}
\DoxyCodeLine{00485\ \ \ \ \ \ \ \ \ n\ =\ 2;}
\DoxyCodeLine{00486\ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00487\ \ \ \ \ \ \ \ \ n\ =\ 1;}
\DoxyCodeLine{00488\ \ \ \ \ \textcolor{comment}{/*\ Initialize\ base\ model.\ */}}
\DoxyCodeLine{00489\ \ \ \ \ ngram\_model\_init(base,\ \&ngram\_model\_arpa\_funcs,\ lmath,\ n,\ n\_unigram);}
\DoxyCodeLine{00490\ \ \ \ \ base-\/>n\_counts[0]\ =\ n\_unigram;}
\DoxyCodeLine{00491\ \ \ \ \ base-\/>n\_counts[1]\ =\ n\_bigram;}
\DoxyCodeLine{00492\ \ \ \ \ base-\/>n\_counts[2]\ =\ n\_trigram;}
\DoxyCodeLine{00493\ \ \ \ \ base-\/>writable\ =\ TRUE;}
\DoxyCodeLine{00494\ }
\DoxyCodeLine{00495\ \ \ \ \ \textcolor{comment}{/*}}
\DoxyCodeLine{00496\ \textcolor{comment}{\ \ \ \ \ *\ Allocate\ one\ extra\ unigram\ and\ bigram\ entry:\ sentinels\ to\ terminate}}
\DoxyCodeLine{00497\ \textcolor{comment}{\ \ \ \ \ *\ followers\ (bigrams\ and\ trigrams,\ respectively)\ of\ previous\ entry.}}
\DoxyCodeLine{00498\ \textcolor{comment}{\ \ \ \ \ */}}
\DoxyCodeLine{00499\ \ \ \ \ model-\/>lm3g.unigrams\ =\ new\_unigram\_table(n\_unigram\ +\ 1);}
\DoxyCodeLine{00500\ \ \ \ \ model-\/>lm3g.bigrams\ =}
\DoxyCodeLine{00501\ \ \ \ \ \ \ \ \ ckd\_calloc(n\_bigram\ +\ 1,\ \textcolor{keyword}{sizeof}(bigram\_t));}
\DoxyCodeLine{00502\ \ \ \ \ \textcolor{keywordflow}{if}\ (n\_trigram\ >\ 0)}
\DoxyCodeLine{00503\ \ \ \ \ \ \ \ \ model-\/>lm3g.trigrams\ =}
\DoxyCodeLine{00504\ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_calloc(n\_trigram,\ \textcolor{keyword}{sizeof}(trigram\_t));}
\DoxyCodeLine{00505\ }
\DoxyCodeLine{00506\ \ \ \ \ \textcolor{keywordflow}{if}\ (n\_trigram\ >\ 0)\ \{}
\DoxyCodeLine{00507\ \ \ \ \ \ \ \ \ model-\/>lm3g.tseg\_base\ =}
\DoxyCodeLine{00508\ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_calloc((n\_bigram\ +\ 1)\ /\ BG\_SEG\_SZ\ +\ 1,}
\DoxyCodeLine{00509\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{sizeof}(int32));}
\DoxyCodeLine{00510\ \ \ \ \ \}}
\DoxyCodeLine{00511\ \ \ \ \ \textcolor{keywordflow}{if}\ (ReadUnigrams(\&li,\ model)\ ==\ -\/1)\ \{}
\DoxyCodeLine{00512\ \ \ \ \ \ \ \ \ fclose\_comp(fp,\ is\_pipe);}
\DoxyCodeLine{00513\ \ \ \ \ \ \ \ \ ngram\_model\_free(base);}
\DoxyCodeLine{00514\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00515\ \ \ \ \ \}}
\DoxyCodeLine{00516\ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ \#unigrams\ created\(\backslash\)n"{}},\ base-\/>n\_counts[0]);}
\DoxyCodeLine{00517\ }
\DoxyCodeLine{00518\ \ \ \ \ init\_sorted\_list(\&model-\/>sorted\_prob2);}
\DoxyCodeLine{00519\ \ \ \ \ \textcolor{keywordflow}{if}\ (base-\/>n\_counts[2]\ >\ 0)}
\DoxyCodeLine{00520\ \ \ \ \ \ \ \ \ init\_sorted\_list(\&model-\/>sorted\_bo\_wt2);}
\DoxyCodeLine{00521\ }
\DoxyCodeLine{00522\ \ \ \ \ \textcolor{keywordflow}{if}\ (base-\/>n\_counts[1]\ >\ 0)\ \{}
\DoxyCodeLine{00523\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (ReadBigrams(\&li,\ model)\ ==\ -\/1)\ \{}
\DoxyCodeLine{00524\ \ \ \ \ \ \ \ \ \ \ \ \ fclose\_comp(fp,\ is\_pipe);}
\DoxyCodeLine{00525\ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_model\_free(base);}
\DoxyCodeLine{00526\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00527\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00528\ }
\DoxyCodeLine{00529\ \ \ \ \ \ \ \ \ base-\/>n\_counts[1]\ =\ FIRST\_BG(model,\ base-\/>n\_counts[0]);}
\DoxyCodeLine{00530\ \ \ \ \ \ \ \ \ model-\/>lm3g.n\_prob2\ =\ model-\/>sorted\_prob2.free;}
\DoxyCodeLine{00531\ \ \ \ \ \ \ \ \ model-\/>lm3g.prob2\ =\ vals\_in\_sorted\_list(\&model-\/>sorted\_prob2);}
\DoxyCodeLine{00532\ \ \ \ \ \ \ \ \ free\_sorted\_list(\&model-\/>sorted\_prob2);}
\DoxyCodeLine{00533\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ \#bigrams\ created\(\backslash\)n"{}},\ base-\/>n\_counts[1]);}
\DoxyCodeLine{00534\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ \#prob2\ entries\(\backslash\)n"{}},\ model-\/>lm3g.n\_prob2);}
\DoxyCodeLine{00535\ \ \ \ \ \}}
\DoxyCodeLine{00536\ }
\DoxyCodeLine{00537\ \ \ \ \ \textcolor{keywordflow}{if}\ (base-\/>n\_counts[2]\ >\ 0)\ \{}
\DoxyCodeLine{00538\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Create\ trigram\ bo-\/wts\ array\ */}}
\DoxyCodeLine{00539\ \ \ \ \ \ \ \ \ model-\/>lm3g.n\_bo\_wt2\ =\ model-\/>sorted\_bo\_wt2.free;}
\DoxyCodeLine{00540\ \ \ \ \ \ \ \ \ model-\/>lm3g.bo\_wt2\ =\ vals\_in\_sorted\_list(\&model-\/>sorted\_bo\_wt2);}
\DoxyCodeLine{00541\ \ \ \ \ \ \ \ \ free\_sorted\_list(\&model-\/>sorted\_bo\_wt2);}
\DoxyCodeLine{00542\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ \#bo\_wt2\ entries\(\backslash\)n"{}},\ model-\/>lm3g.n\_bo\_wt2);}
\DoxyCodeLine{00543\ }
\DoxyCodeLine{00544\ \ \ \ \ \ \ \ \ init\_sorted\_list(\&model-\/>sorted\_prob3);}
\DoxyCodeLine{00545\ }
\DoxyCodeLine{00546\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (ReadTrigrams(\&li,\ model)\ ==\ -\/1)\ \{}
\DoxyCodeLine{00547\ \ \ \ \ \ \ \ \ \ \ \ \ fclose\_comp(fp,\ is\_pipe);}
\DoxyCodeLine{00548\ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_model\_free(base);}
\DoxyCodeLine{00549\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00550\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00551\ }
\DoxyCodeLine{00552\ \ \ \ \ \ \ \ \ base-\/>n\_counts[2]\ =\ FIRST\_TG(model,\ base-\/>n\_counts[1]);}
\DoxyCodeLine{00553\ \ \ \ \ \ \ \ \ model-\/>lm3g.n\_prob3\ =\ model-\/>sorted\_prob3.free;}
\DoxyCodeLine{00554\ \ \ \ \ \ \ \ \ model-\/>lm3g.prob3\ =\ vals\_in\_sorted\_list(\&model-\/>sorted\_prob3);}
\DoxyCodeLine{00555\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ \#trigrams\ created\(\backslash\)n"{}},\ base-\/>n\_counts[2]);}
\DoxyCodeLine{00556\ \ \ \ \ \ \ \ \ E\_INFO(\textcolor{stringliteral}{"{}\%8d\ =\ \#prob3\ entries\(\backslash\)n"{}},\ model-\/>lm3g.n\_prob3);}
\DoxyCodeLine{00557\ }
\DoxyCodeLine{00558\ \ \ \ \ \ \ \ \ free\_sorted\_list(\&model-\/>sorted\_prob3);}
\DoxyCodeLine{00559\ }
\DoxyCodeLine{00560\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Initialize\ tginfo\ */}}
\DoxyCodeLine{00561\ \ \ \ \ \ \ \ \ model-\/>lm3g.tginfo\ =\ ckd\_calloc(n\_unigram,\ \textcolor{keyword}{sizeof}(tginfo\_t\ *));}
\DoxyCodeLine{00562\ \ \ \ \ \ \ \ \ model-\/>lm3g.le\ =\ listelem\_alloc\_init(\textcolor{keyword}{sizeof}(tginfo\_t));}
\DoxyCodeLine{00563\ \ \ \ \ \}}
\DoxyCodeLine{00564\ }
\DoxyCodeLine{00565\ \ \ \ \ lineiter\_free(li);}
\DoxyCodeLine{00566\ \ \ \ \ fclose\_comp(fp,\ is\_pipe);}
\DoxyCodeLine{00567\ \ \ \ \ \textcolor{keywordflow}{return}\ base;}
\DoxyCodeLine{00568\ \}}
\DoxyCodeLine{00569\ }
\DoxyCodeLine{00570\ \textcolor{keywordtype}{int}}
\DoxyCodeLine{00571\ ngram\_model\_arpa\_write(ngram\_model\_t\ *model,}
\DoxyCodeLine{00572\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *file\_name)}
\DoxyCodeLine{00573\ \{}
\DoxyCodeLine{00574\ \ \ \ \ ngram\_iter\_t\ *itor;}
\DoxyCodeLine{00575\ \ \ \ \ FILE\ *fh;}
\DoxyCodeLine{00576\ \ \ \ \ \textcolor{keywordtype}{int}\ i;}
\DoxyCodeLine{00577\ }
\DoxyCodeLine{00578\ \ \ \ \ \textcolor{keywordflow}{if}\ ((fh\ =\ fopen(file\_name,\ \textcolor{stringliteral}{"{}w"{}}))\ ==\ NULL)\ \{}
\DoxyCodeLine{00579\ \ \ \ \ \ \ \ \ E\_ERROR\_SYSTEM(\textcolor{stringliteral}{"{}Failed\ to\ open\ \%s\ for\ writing"{}},\ file\_name);}
\DoxyCodeLine{00580\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00581\ \ \ \ \ \}}
\DoxyCodeLine{00582\ \ \ \ \ fprintf(fh,\ \textcolor{stringliteral}{"{}This\ is\ an\ ARPA-\/format\ language\ model\ file,\ generated\ by\ CMU\ Sphinx\(\backslash\)n"{}});}
\DoxyCodeLine{00583\ }
\DoxyCodeLine{00584\ \ \ \ \ \textcolor{comment}{/*\ The\ ARPA\ format\ doesn't\ require\ any\ extra\ information\ that}}
\DoxyCodeLine{00585\ \textcolor{comment}{\ \ \ \ \ *\ N-\/Gram\ iterators\ can't\ give\ us,\ so\ this\ is\ very}}
\DoxyCodeLine{00586\ \textcolor{comment}{\ \ \ \ \ *\ straightforward\ compared\ with\ DMP\ writing.\ */}}
\DoxyCodeLine{00587\ }
\DoxyCodeLine{00588\ \ \ \ \ \textcolor{comment}{/*\ Write\ N-\/gram\ counts.\ */}}
\DoxyCodeLine{00589\ \ \ \ \ fprintf(fh,\ \textcolor{stringliteral}{"{}\(\backslash\)\(\backslash\)data\(\backslash\)\(\backslash\)\(\backslash\)n"{}});}
\DoxyCodeLine{00590\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ model-\/>n;\ ++i)\ \{}
\DoxyCodeLine{00591\ \ \ \ \ \ \ \ \ fprintf(fh,\ \textcolor{stringliteral}{"{}ngram\ \%d=\%d\(\backslash\)n"{}},\ i+1,\ model-\/>n\_counts[i]);}
\DoxyCodeLine{00592\ \ \ \ \ \}}
\DoxyCodeLine{00593\ }
\DoxyCodeLine{00594\ \ \ \ \ \textcolor{comment}{/*\ Write\ N-\/grams\ */}}
\DoxyCodeLine{00595\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ model-\/>n;\ ++i)\ \{}
\DoxyCodeLine{00596\ \ \ \ \ \ \ \ \ fprintf(fh,\ \textcolor{stringliteral}{"{}\(\backslash\)n\(\backslash\)\(\backslash\)\%d-\/grams:\(\backslash\)n"{}},\ i\ +\ 1);}
\DoxyCodeLine{00597\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (itor\ =\ ngram\_model\_mgrams(model,\ i);\ itor;\ itor\ =\ ngram\_iter\_next(itor))\ \{}
\DoxyCodeLine{00598\ \ \ \ \ \ \ \ \ \ \ \ \ int32\ \textcolor{keyword}{const}\ *wids;}
\DoxyCodeLine{00599\ \ \ \ \ \ \ \ \ \ \ \ \ int32\ score,\ bowt;}
\DoxyCodeLine{00600\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{int}\ j;}
\DoxyCodeLine{00601\ }
\DoxyCodeLine{00602\ \ \ \ \ \ \ \ \ \ \ \ \ wids\ =\ ngram\_iter\_get(itor,\ \&score,\ \&bowt);}
\DoxyCodeLine{00603\ \ \ \ \ \ \ \ \ \ \ \ \ fprintf(fh,\ \textcolor{stringliteral}{"{}\%.4f\ "{}},\ logmath\_log\_to\_log10(model-\/>lmath,\ score));}
\DoxyCodeLine{00604\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (j\ =\ 0;\ j\ <=\ i;\ ++j)\ \{}
\DoxyCodeLine{00605\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ assert(wids[j]\ <\ model-\/>n\_counts[0]);}
\DoxyCodeLine{00606\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ fprintf(fh,\ \textcolor{stringliteral}{"{}\%s\ "{}},\ model-\/>word\_str[wids[j]]);}
\DoxyCodeLine{00607\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00608\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (i\ <\ model-\/>n-\/1)}
\DoxyCodeLine{00609\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ fprintf(fh,\ \textcolor{stringliteral}{"{}\%.4f"{}},\ logmath\_log\_to\_log10(model-\/>lmath,\ bowt));}
\DoxyCodeLine{00610\ \ \ \ \ \ \ \ \ \ \ \ \ fprintf(fh,\ \textcolor{stringliteral}{"{}\(\backslash\)n"{}});}
\DoxyCodeLine{00611\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00612\ \ \ \ \ \}}
\DoxyCodeLine{00613\ \ \ \ \ fprintf(fh,\ \textcolor{stringliteral}{"{}\(\backslash\)n\(\backslash\)\(\backslash\)end\(\backslash\)\(\backslash\)\(\backslash\)n"{}});}
\DoxyCodeLine{00614\ \ \ \ \ \textcolor{keywordflow}{return}\ fclose(fh);}
\DoxyCodeLine{00615\ \}}
\DoxyCodeLine{00616\ }
\DoxyCodeLine{00617\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{int}}
\DoxyCodeLine{00618\ ngram\_model\_arpa\_apply\_weights(ngram\_model\_t\ *base,\ float32\ lw,}
\DoxyCodeLine{00619\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ float32\ wip,\ float32\ uw)}
\DoxyCodeLine{00620\ \{}
\DoxyCodeLine{00621\ \ \ \ \ ngram\_model\_arpa\_t\ *model\ =\ (ngram\_model\_arpa\_t\ *)base;}
\DoxyCodeLine{00622\ \ \ \ \ lm3g\_apply\_weights(base,\ \&model-\/>lm3g,\ lw,\ wip,\ uw);}
\DoxyCodeLine{00623\ \ \ \ \ \textcolor{keywordflow}{return}\ 0;}
\DoxyCodeLine{00624\ \}}
\DoxyCodeLine{00625\ }
\DoxyCodeLine{00626\ \textcolor{comment}{/*\ Lousy\ "{}templating"{}\ for\ things\ that\ are\ largely\ the\ same\ in\ DMP\ and}}
\DoxyCodeLine{00627\ \textcolor{comment}{\ *\ ARPA\ models,\ except\ for\ the\ bigram\ and\ trigram\ types\ and\ some}}
\DoxyCodeLine{00628\ \textcolor{comment}{\ *\ names.\ */}}
\DoxyCodeLine{00629\ \textcolor{preprocessor}{\#define\ NGRAM\_MODEL\_TYPE\ ngram\_model\_arpa\_t}}
\DoxyCodeLine{00630\ \textcolor{preprocessor}{\#include\ "{}lm3g\_templates.c"{}}}
\DoxyCodeLine{00631\ }
\DoxyCodeLine{00632\ \textcolor{keyword}{static}\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00633\ ngram\_model\_arpa\_free(ngram\_model\_t\ *base)}
\DoxyCodeLine{00634\ \{}
\DoxyCodeLine{00635\ \ \ \ \ ngram\_model\_arpa\_t\ *model\ =\ (ngram\_model\_arpa\_t\ *)base;}
\DoxyCodeLine{00636\ \ \ \ \ ckd\_free(model-\/>lm3g.unigrams);}
\DoxyCodeLine{00637\ \ \ \ \ ckd\_free(model-\/>lm3g.bigrams);}
\DoxyCodeLine{00638\ \ \ \ \ ckd\_free(model-\/>lm3g.trigrams);}
\DoxyCodeLine{00639\ \ \ \ \ ckd\_free(model-\/>lm3g.prob2);}
\DoxyCodeLine{00640\ \ \ \ \ ckd\_free(model-\/>lm3g.bo\_wt2);}
\DoxyCodeLine{00641\ \ \ \ \ ckd\_free(model-\/>lm3g.prob3);}
\DoxyCodeLine{00642\ \ \ \ \ lm3g\_tginfo\_free(base,\ \&model-\/>lm3g);}
\DoxyCodeLine{00643\ \ \ \ \ ckd\_free(model-\/>lm3g.tseg\_base);}
\DoxyCodeLine{00644\ \}}
\DoxyCodeLine{00645\ }
\DoxyCodeLine{00646\ \textcolor{keyword}{static}\ ngram\_funcs\_t\ ngram\_model\_arpa\_funcs\ =\ \{}
\DoxyCodeLine{00647\ \ \ \ \ ngram\_model\_arpa\_free,\ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ free\ */}}
\DoxyCodeLine{00648\ \ \ \ \ ngram\_model\_arpa\_apply\_weights,\ \textcolor{comment}{/*\ apply\_weights\ */}}
\DoxyCodeLine{00649\ \ \ \ \ lm3g\_template\_score,\ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ score\ */}}
\DoxyCodeLine{00650\ \ \ \ \ lm3g\_template\_raw\_score,\ \ \ \ \ \ \ \ \textcolor{comment}{/*\ raw\_score\ */}}
\DoxyCodeLine{00651\ \ \ \ \ lm3g\_template\_add\_ug,\ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ add\_ug\ */}}
\DoxyCodeLine{00652\ \ \ \ \ lm3g\_template\_flush,\ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ flush\ */}}
\DoxyCodeLine{00653\ \ \ \ \ lm3g\_template\_iter,\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ iter\ */}}
\DoxyCodeLine{00654\ \ \ \ \ lm3g\_template\_mgrams,\ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ mgrams\ */}}
\DoxyCodeLine{00655\ \ \ \ \ lm3g\_template\_successors,\ \ \ \ \ \ \ \textcolor{comment}{/*\ successors\ */}}
\DoxyCodeLine{00656\ \ \ \ \ lm3g\_template\_iter\_get,\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ iter\_get\ */}}
\DoxyCodeLine{00657\ \ \ \ \ lm3g\_template\_iter\_next,\ \ \ \ \ \ \ \ \textcolor{comment}{/*\ iter\_next\ */}}
\DoxyCodeLine{00658\ \ \ \ \ lm3g\_template\_iter\_free\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ iter\_free\ */}}
\DoxyCodeLine{00659\ \};}

\end{DoxyCode}
