\doxysection{ngram\+\_\+model.\+c}
\label{ngram__model_8c_source}\index{src/libsphinxbase/lm/ngram\_model.c@{src/libsphinxbase/lm/ngram\_model.c}}

\begin{DoxyCode}{0}
\DoxyCodeLine{00001\ \textcolor{comment}{/*\ -\/*-\/\ c-\/basic-\/offset:\ 4;\ indent-\/tabs-\/mode:\ nil\ -\/*-\/\ */}}
\DoxyCodeLine{00002\ \textcolor{comment}{/*\ ====================================================================}}
\DoxyCodeLine{00003\ \textcolor{comment}{\ *\ Copyright\ (c)\ 1999-\/2007\ Carnegie\ Mellon\ University.\ \ All\ rights}}
\DoxyCodeLine{00004\ \textcolor{comment}{\ *\ reserved.}}
\DoxyCodeLine{00005\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00006\ \textcolor{comment}{\ *\ Redistribution\ and\ use\ in\ source\ and\ binary\ forms,\ with\ or\ without}}
\DoxyCodeLine{00007\ \textcolor{comment}{\ *\ modification,\ are\ permitted\ provided\ that\ the\ following\ conditions}}
\DoxyCodeLine{00008\ \textcolor{comment}{\ *\ are\ met:}}
\DoxyCodeLine{00009\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00010\ \textcolor{comment}{\ *\ 1.\ Redistributions\ of\ source\ code\ must\ retain\ the\ above\ copyright}}
\DoxyCodeLine{00011\ \textcolor{comment}{\ *\ \ \ \ notice,\ this\ list\ of\ conditions\ and\ the\ following\ disclaimer.\ }}
\DoxyCodeLine{00012\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00013\ \textcolor{comment}{\ *\ 2.\ Redistributions\ in\ binary\ form\ must\ reproduce\ the\ above\ copyright}}
\DoxyCodeLine{00014\ \textcolor{comment}{\ *\ \ \ \ notice,\ this\ list\ of\ conditions\ and\ the\ following\ disclaimer\ in}}
\DoxyCodeLine{00015\ \textcolor{comment}{\ *\ \ \ \ the\ documentation\ and/or\ other\ materials\ provided\ with\ the}}
\DoxyCodeLine{00016\ \textcolor{comment}{\ *\ \ \ \ distribution.}}
\DoxyCodeLine{00017\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00018\ \textcolor{comment}{\ *\ This\ work\ was\ supported\ in\ part\ by\ funding\ from\ the\ Defense\ Advanced\ }}
\DoxyCodeLine{00019\ \textcolor{comment}{\ *\ Research\ Projects\ Agency\ and\ the\ National\ Science\ Foundation\ of\ the\ }}
\DoxyCodeLine{00020\ \textcolor{comment}{\ *\ United\ States\ of\ America,\ and\ the\ CMU\ Sphinx\ Speech\ Consortium.}}
\DoxyCodeLine{00021\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00022\ \textcolor{comment}{\ *\ THIS\ SOFTWARE\ IS\ PROVIDED\ BY\ CARNEGIE\ MELLON\ UNIVERSITY\ \`{}\`{}AS\ IS''\ AND\ }}
\DoxyCodeLine{00023\ \textcolor{comment}{\ *\ ANY\ EXPRESSED\ OR\ IMPLIED\ WARRANTIES,\ INCLUDING,\ BUT\ NOT\ LIMITED\ TO,\ }}
\DoxyCodeLine{00024\ \textcolor{comment}{\ *\ THE\ IMPLIED\ WARRANTIES\ OF\ MERCHANTABILITY\ AND\ FITNESS\ FOR\ A\ PARTICULAR}}
\DoxyCodeLine{00025\ \textcolor{comment}{\ *\ PURPOSE\ ARE\ DISCLAIMED.\ \ IN\ NO\ EVENT\ SHALL\ CARNEGIE\ MELLON\ UNIVERSITY}}
\DoxyCodeLine{00026\ \textcolor{comment}{\ *\ NOR\ ITS\ EMPLOYEES\ BE\ LIABLE\ FOR\ ANY\ DIRECT,\ INDIRECT,\ INCIDENTAL,}}
\DoxyCodeLine{00027\ \textcolor{comment}{\ *\ SPECIAL,\ EXEMPLARY,\ OR\ CONSEQUENTIAL\ DAMAGES\ (INCLUDING,\ BUT\ NOT\ }}
\DoxyCodeLine{00028\ \textcolor{comment}{\ *\ LIMITED\ TO,\ PROCUREMENT\ OF\ SUBSTITUTE\ GOODS\ OR\ SERVICES;\ LOSS\ OF\ USE,\ }}
\DoxyCodeLine{00029\ \textcolor{comment}{\ *\ DATA,\ OR\ PROFITS;\ OR\ BUSINESS\ INTERRUPTION)\ HOWEVER\ CAUSED\ AND\ ON\ ANY\ }}
\DoxyCodeLine{00030\ \textcolor{comment}{\ *\ THEORY\ OF\ LIABILITY,\ WHETHER\ IN\ CONTRACT,\ STRICT\ LIABILITY,\ OR\ TORT\ }}
\DoxyCodeLine{00031\ \textcolor{comment}{\ *\ (INCLUDING\ NEGLIGENCE\ OR\ OTHERWISE)\ ARISING\ IN\ ANY\ WAY\ OUT\ OF\ THE\ USE\ }}
\DoxyCodeLine{00032\ \textcolor{comment}{\ *\ OF\ THIS\ SOFTWARE,\ EVEN\ IF\ ADVISED\ OF\ THE\ POSSIBILITY\ OF\ SUCH\ DAMAGE.}}
\DoxyCodeLine{00033\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00034\ \textcolor{comment}{\ *\ ====================================================================}}
\DoxyCodeLine{00035\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00036\ \textcolor{comment}{\ */}}
\DoxyCodeLine{00037\ \textcolor{comment}{/*}}
\DoxyCodeLine{00038\ \textcolor{comment}{\ *\ \(\backslash\)file\ ngram\_model.c\ N-\/Gram\ language\ models.}}
\DoxyCodeLine{00039\ \textcolor{comment}{\ *}}
\DoxyCodeLine{00040\ \textcolor{comment}{\ *\ Author:\ David\ Huggins-\/Daines,\ much\ code\ taken\ from\ sphinx3/src/libs3decoder/liblm}}
\DoxyCodeLine{00041\ \textcolor{comment}{\ */}}
\DoxyCodeLine{00042\ }
\DoxyCodeLine{00043\ \textcolor{preprocessor}{\#include\ <config.h>}}
\DoxyCodeLine{00044\ }
\DoxyCodeLine{00045\ \textcolor{preprocessor}{\#include\ <string.h>}}
\DoxyCodeLine{00046\ \textcolor{preprocessor}{\#include\ <assert.h>}}
\DoxyCodeLine{00047\ }
\DoxyCodeLine{00048\ \textcolor{preprocessor}{\#ifdef\ HAVE\_ICONV}}
\DoxyCodeLine{00049\ \textcolor{preprocessor}{\#include\ <iconv.h>}}
\DoxyCodeLine{00050\ \textcolor{preprocessor}{\#endif\ }}
\DoxyCodeLine{00051\ }
\DoxyCodeLine{00052\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/ngram\_model.h"{}}}
\DoxyCodeLine{00053\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/ckd\_alloc.h"{}}}
\DoxyCodeLine{00054\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/filename.h"{}}}
\DoxyCodeLine{00055\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/pio.h"{}}}
\DoxyCodeLine{00056\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/err.h"{}}}
\DoxyCodeLine{00057\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/logmath.h"{}}}
\DoxyCodeLine{00058\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/strfuncs.h"{}}}
\DoxyCodeLine{00059\ \textcolor{preprocessor}{\#include\ "{}sphinxbase/case.h"{}}}
\DoxyCodeLine{00060\ }
\DoxyCodeLine{00061\ \textcolor{preprocessor}{\#include\ "{}ngram\_model\_internal.h"{}}}
\DoxyCodeLine{00062\ }
\DoxyCodeLine{00063\ ngram\_file\_type\_t}
\DoxyCodeLine{00064\ ngram\_file\_name\_to\_type(\textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *file\_name)}
\DoxyCodeLine{00065\ \{}
\DoxyCodeLine{00066\ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *ext;}
\DoxyCodeLine{00067\ }
\DoxyCodeLine{00068\ \ \ \ \ ext\ =\ strrchr(file\_name,\ \textcolor{charliteral}{'.'});}
\DoxyCodeLine{00069\ \ \ \ \ \textcolor{keywordflow}{if}\ (ext\ ==\ NULL)\ \{}
\DoxyCodeLine{00070\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NGRAM\_INVALID;}
\DoxyCodeLine{00071\ \ \ \ \ \}}
\DoxyCodeLine{00072\ \ \ \ \ \textcolor{keywordflow}{if}\ (0\ ==\ strcmp\_nocase(ext,\ \textcolor{stringliteral}{"{}.gz"{}}))\ \{}
\DoxyCodeLine{00073\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{while}\ (-\/-\/ext\ >=\ file\_name)\ \{}
\DoxyCodeLine{00074\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (*ext\ ==\ \textcolor{charliteral}{'.'})\ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00075\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00076\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (ext\ <\ file\_name)\ \{}
\DoxyCodeLine{00077\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NGRAM\_INVALID;}
\DoxyCodeLine{00078\ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00079\ \ \ \ \ \ \}}
\DoxyCodeLine{00080\ \ \ \ \ \ \textcolor{keywordflow}{else}\ \textcolor{keywordflow}{if}\ (0\ ==\ strcmp\_nocase(ext,\ \textcolor{stringliteral}{"{}.bz2"{}}))\ \{}
\DoxyCodeLine{00081\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{while}\ (-\/-\/ext\ >=\ file\_name)\ \{}
\DoxyCodeLine{00082\ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (*ext\ ==\ \textcolor{charliteral}{'.'})\ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00083\ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00084\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (ext\ <\ file\_name)\ \{}
\DoxyCodeLine{00085\ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NGRAM\_INVALID;}
\DoxyCodeLine{00086\ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00087\ \ \ \ \ \ \}}
\DoxyCodeLine{00088\ \ \ \ \ \ \textcolor{comment}{/*\ We\ use\ strncmp\ because\ there\ might\ be\ a\ .gz\ on\ the\ end.\ */}}
\DoxyCodeLine{00089\ \ \ \ \ \ \textcolor{keywordflow}{if}\ (0\ ==\ strncmp\_nocase(ext,\ \textcolor{stringliteral}{"{}.ARPA"{}},\ 5))}
\DoxyCodeLine{00090\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NGRAM\_ARPA;}
\DoxyCodeLine{00091\ \ \ \ \ \ \textcolor{keywordflow}{if}\ (0\ ==\ strncmp\_nocase(ext,\ \textcolor{stringliteral}{"{}.DMP"{}},\ 4))}
\DoxyCodeLine{00092\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NGRAM\_DMP;}
\DoxyCodeLine{00093\ \ \ \ \ \ \textcolor{keywordflow}{return}\ NGRAM\_INVALID;}
\DoxyCodeLine{00094\ \ \}}
\DoxyCodeLine{00095\ }
\DoxyCodeLine{00096\ ngram\_file\_type\_t}
\DoxyCodeLine{00097\ ngram\_str\_to\_type(\textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *str\_name)}
\DoxyCodeLine{00098\ \{}
\DoxyCodeLine{00099\ \ \ \ \ \textcolor{keywordflow}{if}\ (0\ ==\ strcmp\_nocase(str\_name,\ \textcolor{stringliteral}{"{}arpa"{}}))}
\DoxyCodeLine{00100\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NGRAM\_ARPA;}
\DoxyCodeLine{00101\ \ \ \ \ \textcolor{keywordflow}{if}\ (0\ ==\ strcmp\_nocase(str\_name,\ \textcolor{stringliteral}{"{}dmp"{}}))}
\DoxyCodeLine{00102\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NGRAM\_DMP;}
\DoxyCodeLine{00103\ \ \ \ \ \textcolor{keywordflow}{return}\ NGRAM\_INVALID;}
\DoxyCodeLine{00104\ \}}
\DoxyCodeLine{00105\ }
\DoxyCodeLine{00106\ \textcolor{keywordtype}{char}\ \textcolor{keyword}{const}\ *}
\DoxyCodeLine{00107\ ngram\_type\_to\_str(\textcolor{keywordtype}{int}\ type)}
\DoxyCodeLine{00108\ \{}
\DoxyCodeLine{00109\ \ \ \ \ \textcolor{keywordflow}{switch}\ (type)\ \{}
\DoxyCodeLine{00110\ \ \ \ \ \textcolor{keywordflow}{case}\ NGRAM\_ARPA:}
\DoxyCodeLine{00111\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ \textcolor{stringliteral}{"{}arpa"{}};}
\DoxyCodeLine{00112\ \ \ \ \ \textcolor{keywordflow}{case}\ NGRAM\_DMP:}
\DoxyCodeLine{00113\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ \textcolor{stringliteral}{"{}dmp"{}};}
\DoxyCodeLine{00114\ \ \ \ \ \textcolor{keywordflow}{default}:}
\DoxyCodeLine{00115\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00116\ \ \ \ \ \}}
\DoxyCodeLine{00117\ \}}
\DoxyCodeLine{00118\ }
\DoxyCodeLine{00119\ }
\DoxyCodeLine{00120\ \ ngram\_model\_t\ *}
\DoxyCodeLine{00121\ \ ngram\_model\_read(cmd\_ln\_t\ *config,}
\DoxyCodeLine{00122\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *file\_name,}
\DoxyCodeLine{00123\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_file\_type\_t\ file\_type,}
\DoxyCodeLine{00124\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ logmath\_t\ *lmath)}
\DoxyCodeLine{00125\ \ \{}
\DoxyCodeLine{00126\ \ \ \ \ \ ngram\_model\_t\ *model\ =\ NULL;}
\DoxyCodeLine{00127\ }
\DoxyCodeLine{00128\ \ \ \ \ \ \textcolor{keywordflow}{switch}\ (file\_type)\ \{}
\DoxyCodeLine{00129\ \ \ \ \ \ \textcolor{keywordflow}{case}\ NGRAM\_AUTO:\ \{}
\DoxyCodeLine{00130\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((model\ =\ ngram\_model\_arpa\_read(config,\ file\_name,\ lmath))\ !=\ NULL)}
\DoxyCodeLine{00131\ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00132\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((model\ =\ ngram\_model\_dmp\_read(config,\ file\_name,\ lmath))\ !=\ NULL)}
\DoxyCodeLine{00133\ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00134\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00135\ \ \ \ \ \ \}}
\DoxyCodeLine{00136\ \ \ \ \ \ \textcolor{keywordflow}{case}\ NGRAM\_ARPA:}
\DoxyCodeLine{00137\ \ \ \ \ \ \ \ \ \ model\ =\ ngram\_model\_arpa\_read(config,\ file\_name,\ lmath);}
\DoxyCodeLine{00138\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00139\ \ \ \ \ \ \textcolor{keywordflow}{case}\ NGRAM\_DMP:}
\DoxyCodeLine{00140\ \ \ \ \ \ \ \ \ \ model\ =\ ngram\_model\_dmp\_read(config,\ file\_name,\ lmath);}
\DoxyCodeLine{00141\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00142\ \ \ \ \ \ \textcolor{keywordflow}{default}:}
\DoxyCodeLine{00143\ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}language\ model\ file\ type\ not\ supported\(\backslash\)n"{}});}
\DoxyCodeLine{00144\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00145\ \ \ \ \ \ \}}
\DoxyCodeLine{00146\ }
\DoxyCodeLine{00147\ \ \ \ \ \ \textcolor{comment}{/*\ Now\ set\ weights\ based\ on\ config\ if\ present.\ */}}
\DoxyCodeLine{00148\ \ \ \ \ \ \textcolor{keywordflow}{if}\ (config)\ \{}
\DoxyCodeLine{00149\ \ \ \ \ \ \ \ \ \ float32\ lw\ =\ 1.0;}
\DoxyCodeLine{00150\ \ \ \ \ \ \ \ \ \ float32\ wip\ =\ 1.0;}
\DoxyCodeLine{00151\ \ \ \ \ \ \ \ \ \ float32\ uw\ =\ 1.0;}
\DoxyCodeLine{00152\ }
\DoxyCodeLine{00153\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (cmd\_ln\_exists\_r(config,\ \textcolor{stringliteral}{"{}-\/lw"{}}))}
\DoxyCodeLine{00154\ \ \ \ \ \ \ \ \ \ \ \ \ \ lw\ =\ cmd\_ln\_float32\_r(config,\ \textcolor{stringliteral}{"{}-\/lw"{}});}
\DoxyCodeLine{00155\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (cmd\_ln\_exists\_r(config,\ \textcolor{stringliteral}{"{}-\/wip"{}}))}
\DoxyCodeLine{00156\ \ \ \ \ \ \ \ \ \ \ \ \ \ wip\ =\ cmd\_ln\_float32\_r(config,\ \textcolor{stringliteral}{"{}-\/wip"{}});}
\DoxyCodeLine{00157\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (cmd\_ln\_exists\_r(config,\ \textcolor{stringliteral}{"{}-\/uw"{}}))}
\DoxyCodeLine{00158\ \ \ \ \ \ \ \ \ \ \ \ \ \ uw\ =\ cmd\_ln\_float32\_r(config,\ \textcolor{stringliteral}{"{}-\/uw"{}});}
\DoxyCodeLine{00159\ }
\DoxyCodeLine{00160\ \ \ \ \ \ \ \ \ \ ngram\_model\_apply\_weights(model,\ lw,\ wip,\ uw);}
\DoxyCodeLine{00161\ \ \ \ \ \ \}}
\DoxyCodeLine{00162\ }
\DoxyCodeLine{00163\ \ \ \ \ \ \textcolor{keywordflow}{return}\ model;}
\DoxyCodeLine{00164\ \ \}}
\DoxyCodeLine{00165\ }
\DoxyCodeLine{00166\ \ \textcolor{keywordtype}{int}}
\DoxyCodeLine{00167\ \ ngram\_model\_write(ngram\_model\_t\ *model,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *file\_name,}
\DoxyCodeLine{00168\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_file\_type\_t\ file\_type)}
\DoxyCodeLine{00169\ \ \{}
\DoxyCodeLine{00170\ \ \ \ \ \ \textcolor{keywordflow}{switch}\ (file\_type)\ \{}
\DoxyCodeLine{00171\ \ \ \ \ \ \textcolor{keywordflow}{case}\ NGRAM\_AUTO:\ \{}
\DoxyCodeLine{00172\ \ \ \ \ \ \ \ \ \ file\_type\ =\ ngram\_file\_name\_to\_type(file\_name);}
\DoxyCodeLine{00173\ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Default\ to\ ARPA\ (catches\ .lm\ and\ other\ things)\ */}}
\DoxyCodeLine{00174\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (file\_type\ ==\ NGRAM\_INVALID)}
\DoxyCodeLine{00175\ \ \ \ \ \ \ \ \ \ \ \ \ \ file\_type\ =\ NGRAM\_ARPA;}
\DoxyCodeLine{00176\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ ngram\_model\_write(model,\ file\_name,\ file\_type);}
\DoxyCodeLine{00177\ \ \ \ \ \ \}}
\DoxyCodeLine{00178\ \ \ \ \ \ \textcolor{keywordflow}{case}\ NGRAM\_ARPA:}
\DoxyCodeLine{00179\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ ngram\_model\_arpa\_write(model,\ file\_name);}
\DoxyCodeLine{00180\ \ \ \ \ \ \textcolor{keywordflow}{case}\ NGRAM\_DMP:}
\DoxyCodeLine{00181\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ ngram\_model\_dmp\_write(model,\ file\_name);}
\DoxyCodeLine{00182\ \ \ \ \ \ \textcolor{keywordflow}{default}:}
\DoxyCodeLine{00183\ \ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}language\ model\ file\ type\ not\ supported\(\backslash\)n"{}});}
\DoxyCodeLine{00184\ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00185\ \ \ \ \ \ \}}
\DoxyCodeLine{00186\ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}language\ model\ file\ type\ not\ supported\(\backslash\)n"{}});}
\DoxyCodeLine{00187\ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00188\ \ \}}
\DoxyCodeLine{00189\ }
\DoxyCodeLine{00190\ \ int32}
\DoxyCodeLine{00191\ \ ngram\_model\_init(ngram\_model\_t\ *base,}
\DoxyCodeLine{00192\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_funcs\_t\ *funcs,}
\DoxyCodeLine{00193\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ logmath\_t\ *lmath,}
\DoxyCodeLine{00194\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ n,\ int32\ n\_unigram)}
\DoxyCodeLine{00195\ \ \{}
\DoxyCodeLine{00196\ \ \ \ \ \ base-\/>refcount\ =\ 1;}
\DoxyCodeLine{00197\ \ \ \ \ \ base-\/>funcs\ =\ funcs;}
\DoxyCodeLine{00198\ \ \ \ \ \ base-\/>n\ =\ n;}
\DoxyCodeLine{00199\ \ \ \ \ \ \textcolor{comment}{/*\ If\ this\ was\ previously\ initialized...\ */}}
\DoxyCodeLine{00200\ \ \ \ \ \textcolor{keywordflow}{if}\ (base-\/>n\_counts\ ==\ NULL)}
\DoxyCodeLine{00201\ \ \ \ \ \ \ \ \ base-\/>n\_counts\ =\ ckd\_calloc(3,\ \textcolor{keyword}{sizeof}(*base-\/>n\_counts));}
\DoxyCodeLine{00202\ \ \ \ \ \textcolor{comment}{/*\ Don't\ reset\ weights\ if\ logmath\ object\ hasn't\ changed.\ */}}
\DoxyCodeLine{00203\ \ \ \ \ \textcolor{keywordflow}{if}\ (base-\/>lmath\ !=\ lmath)\ \{}
\DoxyCodeLine{00204\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Set\ default\ values\ for\ weights.\ */}}
\DoxyCodeLine{00205\ \ \ \ \ \ \ \ \ base-\/>lw\ =\ 1.0;}
\DoxyCodeLine{00206\ \ \ \ \ \ \ \ \ base-\/>log\_wip\ =\ 0;\ \textcolor{comment}{/*\ i.e.\ 1.0\ */}}
\DoxyCodeLine{00207\ \ \ \ \ \ \ \ \ base-\/>log\_uw\ =\ 0;\ \ \textcolor{comment}{/*\ i.e.\ 1.0\ */}}
\DoxyCodeLine{00208\ \ \ \ \ \ \ \ \ base-\/>log\_uniform\ =\ logmath\_log(lmath,\ 1.0\ /\ n\_unigram);}
\DoxyCodeLine{00209\ \ \ \ \ \ \ \ \ base-\/>log\_uniform\_weight\ =\ logmath\_get\_zero(lmath);}
\DoxyCodeLine{00210\ \ \ \ \ \ \ \ \ base-\/>log\_zero\ =\ logmath\_get\_zero(lmath);}
\DoxyCodeLine{00211\ \ \ \ \ \ \ \ \ base-\/>lmath\ =\ lmath;}
\DoxyCodeLine{00212\ \ \ \ \ \}}
\DoxyCodeLine{00213\ \ \ \ \ \textcolor{comment}{/*\ Allocate\ or\ reallocate\ space\ for\ word\ strings.\ */}}
\DoxyCodeLine{00214\ \ \ \ \ \textcolor{keywordflow}{if}\ (base-\/>word\_str)\ \{}
\DoxyCodeLine{00215\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Free\ all\ previous\ word\ strings\ if\ they\ were\ allocated.\ */}}
\DoxyCodeLine{00216\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (base-\/>writable)\ \{}
\DoxyCodeLine{00217\ \ \ \ \ \ \ \ \ \ \ \ \ int32\ i;}
\DoxyCodeLine{00218\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ base-\/>n\_words;\ ++i)\ \{}
\DoxyCodeLine{00219\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_free(base-\/>word\_str[i]);}
\DoxyCodeLine{00220\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ base-\/>word\_str[i]\ =\ NULL;}
\DoxyCodeLine{00221\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00222\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00223\ \ \ \ \ \ \ \ \ base-\/>word\_str\ =\ ckd\_realloc(base-\/>word\_str,\ n\_unigram\ *\ \textcolor{keyword}{sizeof}(\textcolor{keywordtype}{char}\ *));}
\DoxyCodeLine{00224\ \ \ \ \ \}}
\DoxyCodeLine{00225\ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00226\ \ \ \ \ \ \ \ \ base-\/>word\_str\ =\ ckd\_calloc(n\_unigram,\ \textcolor{keyword}{sizeof}(\textcolor{keywordtype}{char}\ *));}
\DoxyCodeLine{00227\ \ \ \ \ \textcolor{comment}{/*\ NOTE:\ They\ are\ no\ longer\ case-\/insensitive\ since\ we\ are\ allowing}}
\DoxyCodeLine{00228\ \textcolor{comment}{\ \ \ \ \ *\ other\ encodings\ for\ word\ strings.\ \ Beware.\ */}}
\DoxyCodeLine{00229\ \ \ \ \ \textcolor{keywordflow}{if}\ (base-\/>wid)}
\DoxyCodeLine{00230\ \ \ \ \ \ \ \ \ hash\_table\_empty(base-\/>wid);}
\DoxyCodeLine{00231\ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00232\ \ \ \ \ \ \ \ \ base-\/>wid\ =\ hash\_table\_new(n\_unigram,\ FALSE);}
\DoxyCodeLine{00233\ \ \ \ \ base-\/>n\_counts[0]\ =\ base-\/>n\_1g\_alloc\ =\ base-\/>n\_words\ =\ n\_unigram;}
\DoxyCodeLine{00234\ }
\DoxyCodeLine{00235\ \ \ \ \ \textcolor{keywordflow}{return}\ 0;}
\DoxyCodeLine{00236\ \}}
\DoxyCodeLine{00237\ }
\DoxyCodeLine{00238\ ngram\_model\_t\ *}
\DoxyCodeLine{00239\ ngram\_model\_retain(ngram\_model\_t\ *model)}
\DoxyCodeLine{00240\ \{}
\DoxyCodeLine{00241\ \ \ \ \ ++model-\/>refcount;}
\DoxyCodeLine{00242\ \ \ \ \ \textcolor{keywordflow}{return}\ model;}
\DoxyCodeLine{00243\ \}}
\DoxyCodeLine{00244\ }
\DoxyCodeLine{00245\ }
\DoxyCodeLine{00246\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00247\ ngram\_model\_flush(ngram\_model\_t\ *model)}
\DoxyCodeLine{00248\ \{}
\DoxyCodeLine{00249\ \ \ \ \ \textcolor{keywordflow}{if}\ (model-\/>funcs\ \&\&\ model-\/>funcs-\/>flush)}
\DoxyCodeLine{00250\ \ \ \ \ \ \ \ \ (*model-\/>funcs-\/>flush)(model);}
\DoxyCodeLine{00251\ \}}
\DoxyCodeLine{00252\ }
\DoxyCodeLine{00253\ \textcolor{keywordtype}{int}}
\DoxyCodeLine{00254\ ngram\_model\_free(ngram\_model\_t\ *model)}
\DoxyCodeLine{00255\ \{}
\DoxyCodeLine{00256\ \ \ \ \ \textcolor{keywordtype}{int}\ i;}
\DoxyCodeLine{00257\ }
\DoxyCodeLine{00258\ \ \ \ \ \textcolor{keywordflow}{if}\ (model\ ==\ NULL)}
\DoxyCodeLine{00259\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ 0;}
\DoxyCodeLine{00260\ \ \ \ \ \textcolor{keywordflow}{if}\ (-\/-\/model-\/>refcount\ >\ 0)}
\DoxyCodeLine{00261\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ model-\/>refcount;}
\DoxyCodeLine{00262\ \ \ \ \ \textcolor{keywordflow}{if}\ (model-\/>funcs\ \&\&\ model-\/>funcs-\/>free)}
\DoxyCodeLine{00263\ \ \ \ \ \ \ \ \ (*model-\/>funcs-\/>free)(model);}
\DoxyCodeLine{00264\ \ \ \ \ \textcolor{keywordflow}{if}\ (model-\/>writable)\ \{}
\DoxyCodeLine{00265\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Free\ all\ words.\ */}}
\DoxyCodeLine{00266\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ model-\/>n\_words;\ ++i)\ \{}
\DoxyCodeLine{00267\ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_free(model-\/>word\_str[i]);}
\DoxyCodeLine{00268\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00269\ \ \ \ \ \}}
\DoxyCodeLine{00270\ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00271\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Free\ all\ class\ words.\ */}}
\DoxyCodeLine{00272\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ model-\/>n\_classes;\ ++i)\ \{}
\DoxyCodeLine{00273\ \ \ \ \ \ \ \ \ \ \ \ \ ngram\_class\_t\ *lmclass;}
\DoxyCodeLine{00274\ \ \ \ \ \ \ \ \ \ \ \ \ int32\ j;}
\DoxyCodeLine{00275\ }
\DoxyCodeLine{00276\ \ \ \ \ \ \ \ \ \ \ \ \ lmclass\ =\ model-\/>classes[i];}
\DoxyCodeLine{00277\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (j\ =\ 0;\ j\ <\ lmclass-\/>n\_words;\ ++j)\ \{}
\DoxyCodeLine{00278\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_free(model-\/>word\_str[lmclass-\/>start\_wid\ +\ j]);}
\DoxyCodeLine{00279\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00280\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (j\ =\ 0;\ j\ <\ lmclass-\/>n\_hash;\ ++j)\ \{}
\DoxyCodeLine{00281\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (lmclass-\/>nword\_hash[j].wid\ !=\ -\/1)\ \{}
\DoxyCodeLine{00282\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_free(model-\/>word\_str[lmclass-\/>nword\_hash[j].wid]);}
\DoxyCodeLine{00283\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00284\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00285\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00286\ \ \ \ \ \}}
\DoxyCodeLine{00287\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ model-\/>n\_classes;\ ++i)\ \{}
\DoxyCodeLine{00288\ \ \ \ \ \ \ \ \ ngram\_class\_free(model-\/>classes[i]);}
\DoxyCodeLine{00289\ \ \ \ \ \}}
\DoxyCodeLine{00290\ \ \ \ \ ckd\_free(model-\/>classes);}
\DoxyCodeLine{00291\ \ \ \ \ hash\_table\_free(model-\/>wid);}
\DoxyCodeLine{00292\ \ \ \ \ ckd\_free(model-\/>word\_str);}
\DoxyCodeLine{00293\ \ \ \ \ ckd\_free(model-\/>n\_counts);}
\DoxyCodeLine{00294\ \ \ \ \ ckd\_free(model);}
\DoxyCodeLine{00295\ \ \ \ \ \textcolor{keywordflow}{return}\ 0;}
\DoxyCodeLine{00296\ \}}
\DoxyCodeLine{00297\ }
\DoxyCodeLine{00298\ \textcolor{keywordtype}{int}}
\DoxyCodeLine{00299\ ngram\_model\_casefold(ngram\_model\_t\ *model,\ \textcolor{keywordtype}{int}\ kase)}
\DoxyCodeLine{00300\ \{}
\DoxyCodeLine{00301\ \ \ \ \ \textcolor{keywordtype}{int}\ writable,\ i;}
\DoxyCodeLine{00302\ \ \ \ \ hash\_table\_t\ *new\_wid;}
\DoxyCodeLine{00303\ }
\DoxyCodeLine{00304\ \ \ \ \ \textcolor{comment}{/*\ Were\ word\ strings\ already\ allocated?\ */}}
\DoxyCodeLine{00305\ \ \ \ \ writable\ =\ model-\/>writable;}
\DoxyCodeLine{00306\ \ \ \ \ \textcolor{comment}{/*\ Either\ way,\ we\ are\ going\ to\ allocate\ some\ word\ strings.\ */}}
\DoxyCodeLine{00307\ \ \ \ \ model-\/>writable\ =\ TRUE;}
\DoxyCodeLine{00308\ }
\DoxyCodeLine{00309\ \ \ \ \ \textcolor{comment}{/*\ And,\ don't\ forget,\ we\ need\ to\ rebuild\ the\ word\ to\ unigram\ ID}}
\DoxyCodeLine{00310\ \textcolor{comment}{\ \ \ \ \ *\ mapping.\ */}}
\DoxyCodeLine{00311\ \ \ \ \ new\_wid\ =\ hash\_table\_new(model-\/>n\_words,\ FALSE);}
\DoxyCodeLine{00312\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ model-\/>n\_words;\ ++i)\ \{}
\DoxyCodeLine{00313\ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{char}\ *outstr;}
\DoxyCodeLine{00314\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (writable)\ \{}
\DoxyCodeLine{00315\ \ \ \ \ \ \ \ \ \ \ \ \ outstr\ =\ model-\/>word\_str[i];}
\DoxyCodeLine{00316\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00317\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00318\ \ \ \ \ \ \ \ \ \ \ \ \ outstr\ =\ ckd\_salloc(model-\/>word\_str[i]);}
\DoxyCodeLine{00319\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00320\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Don't\ case-\/fold\ <tags>\ or\ [classes]\ */}}
\DoxyCodeLine{00321\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (outstr[0]\ ==\ \textcolor{charliteral}{'<'}\ ||\ outstr[0]\ ==\ \textcolor{charliteral}{'['})\ \{}
\DoxyCodeLine{00322\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00323\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00324\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{switch}\ (kase)\ \{}
\DoxyCodeLine{00325\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{case}\ NGRAM\_UPPER:}
\DoxyCodeLine{00326\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ucase(outstr);}
\DoxyCodeLine{00327\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00328\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{case}\ NGRAM\_LOWER:}
\DoxyCodeLine{00329\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ lcase(outstr);}
\DoxyCodeLine{00330\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00331\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{default}:}
\DoxyCodeLine{00332\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ;}
\DoxyCodeLine{00333\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00334\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00335\ \ \ \ \ \ \ \ \ model-\/>word\_str[i]\ =\ outstr;}
\DoxyCodeLine{00336\ }
\DoxyCodeLine{00337\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Now\ update\ the\ hash\ table.\ \ We\ might\ have\ terrible}}
\DoxyCodeLine{00338\ \textcolor{comment}{\ \ \ \ \ \ \ \ \ *\ collisions\ here,\ so\ warn\ about\ them.\ */}}
\DoxyCodeLine{00339\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (hash\_table\_enter\_int32(new\_wid,\ model-\/>word\_str[i],\ i)\ !=\ i)\ \{}
\DoxyCodeLine{00340\ \ \ \ \ \ \ \ \ \ \ \ \ E\_WARN(\textcolor{stringliteral}{"{}Duplicate\ word\ in\ dictionary\ after\ conversion:\ \%s\(\backslash\)n"{}},}
\DoxyCodeLine{00341\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>word\_str[i]);}
\DoxyCodeLine{00342\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00343\ \ \ \ \ \}}
\DoxyCodeLine{00344\ \ \ \ \ \textcolor{comment}{/*\ Swap\ out\ the\ hash\ table.\ */}}
\DoxyCodeLine{00345\ \ \ \ \ hash\_table\_free(model-\/>wid);}
\DoxyCodeLine{00346\ \ \ \ \ model-\/>wid\ =\ new\_wid;}
\DoxyCodeLine{00347\ \ \ \ \ \textcolor{keywordflow}{return}\ 0;}
\DoxyCodeLine{00348\ \}}
\DoxyCodeLine{00349\ }
\DoxyCodeLine{00350\ \textcolor{preprocessor}{\#ifdef\ HAVE\_ICONV}}
\DoxyCodeLine{00351\ \textcolor{keywordtype}{int}}
\DoxyCodeLine{00352\ ngram\_model\_recode(ngram\_model\_t\ *model,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *from,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *to)}
\DoxyCodeLine{00353\ \{}
\DoxyCodeLine{00354\ \ \ \ \ iconv\_t\ ic;}
\DoxyCodeLine{00355\ \ \ \ \ \textcolor{keywordtype}{char}\ *outbuf;}
\DoxyCodeLine{00356\ \ \ \ \ \textcolor{keywordtype}{size\_t}\ maxlen;}
\DoxyCodeLine{00357\ \ \ \ \ \textcolor{keywordtype}{int}\ i,\ writable;}
\DoxyCodeLine{00358\ \ \ \ \ hash\_table\_t\ *new\_wid;}
\DoxyCodeLine{00359\ }
\DoxyCodeLine{00360\ \ \ \ \ \textcolor{comment}{/*\ FIXME:\ Need\ to\ do\ a\ special\ case\ thing\ for\ the\ GB-\/HEX\ encoding}}
\DoxyCodeLine{00361\ \textcolor{comment}{\ \ \ \ \ *\ used\ in\ Sphinx3\ Mandarin\ models.\ */}}
\DoxyCodeLine{00362\ \ \ \ \ \textcolor{keywordflow}{if}\ ((ic\ =\ iconv\_open(to,\ from))\ ==\ (iconv\_t)-\/1)\ \{}
\DoxyCodeLine{00363\ \ \ \ \ \ \ \ \ E\_ERROR\_SYSTEM(\textcolor{stringliteral}{"{}iconv\_open()\ failed"{}});}
\DoxyCodeLine{00364\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00365\ \ \ \ \ \}}
\DoxyCodeLine{00366\ \ \ \ \ \textcolor{comment}{/*\ iconv(3)\ is\ a\ piece\ of\ crap\ and\ won't\ accept\ a\ NULL\ out\ buffer,}}
\DoxyCodeLine{00367\ \textcolor{comment}{\ \ \ \ \ *\ unlike\ wcstombs(3).\ So\ we\ have\ to\ either\ call\ it\ over\ and\ over}}
\DoxyCodeLine{00368\ \textcolor{comment}{\ \ \ \ \ *\ again\ until\ our\ buffer\ is\ big\ enough,\ or\ call\ it\ with\ a\ huge}}
\DoxyCodeLine{00369\ \textcolor{comment}{\ \ \ \ \ *\ buffer\ and\ then\ copy\ things\ back\ to\ the\ output.\ \ We\ will\ use\ a}}
\DoxyCodeLine{00370\ \textcolor{comment}{\ \ \ \ \ *\ mix\ of\ these\ two\ approaches\ here.\ \ We'll\ keep\ a\ single\ big}}
\DoxyCodeLine{00371\ \textcolor{comment}{\ \ \ \ \ *\ buffer\ around,\ and\ expand\ it\ as\ necessary.}}
\DoxyCodeLine{00372\ \textcolor{comment}{\ \ \ \ \ */}}
\DoxyCodeLine{00373\ \ \ \ \ maxlen\ =\ 0;}
\DoxyCodeLine{00374\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ model-\/>n\_words;\ ++i)\ \{}
\DoxyCodeLine{00375\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (strlen(model-\/>word\_str[i])\ >\ maxlen)}
\DoxyCodeLine{00376\ \ \ \ \ \ \ \ \ \ \ \ \ maxlen\ =\ strlen(model-\/>word\_str[i]);}
\DoxyCodeLine{00377\ \ \ \ \ \}}
\DoxyCodeLine{00378\ \ \ \ \ \textcolor{comment}{/*\ Were\ word\ strings\ already\ allocated?\ */}}
\DoxyCodeLine{00379\ \ \ \ \ writable\ =\ model-\/>writable;}
\DoxyCodeLine{00380\ \ \ \ \ \textcolor{comment}{/*\ Either\ way,\ we\ are\ going\ to\ allocate\ some\ word\ strings.\ */}}
\DoxyCodeLine{00381\ \ \ \ \ model-\/>writable\ =\ TRUE;}
\DoxyCodeLine{00382\ \ \ \ \ \textcolor{comment}{/*\ Really\ should\ be\ big\ enough\ except\ for\ pathological\ cases.\ */}}
\DoxyCodeLine{00383\ \ \ \ \ maxlen\ =\ maxlen\ *\ \textcolor{keyword}{sizeof}(int)\ +\ 15;}
\DoxyCodeLine{00384\ \ \ \ \ outbuf\ =\ ckd\_calloc(maxlen,\ 1);}
\DoxyCodeLine{00385\ \ \ \ \ \textcolor{comment}{/*\ And,\ don't\ forget,\ we\ need\ to\ rebuild\ the\ word\ to\ unigram\ ID}}
\DoxyCodeLine{00386\ \textcolor{comment}{\ \ \ \ \ *\ mapping.\ */}}
\DoxyCodeLine{00387\ \ \ \ \ new\_wid\ =\ hash\_table\_new(model-\/>n\_words,\ FALSE);}
\DoxyCodeLine{00388\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ model-\/>n\_words;\ ++i)\ \{}
\DoxyCodeLine{00389\ \ \ \ \ \ \ \ \ ICONV\_CONST\ \textcolor{keywordtype}{char}\ *in;}
\DoxyCodeLine{00390\ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{char}\ *out;}
\DoxyCodeLine{00391\ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{size\_t}\ inleft,\ outleft,\ result;}
\DoxyCodeLine{00392\ }
\DoxyCodeLine{00393\ \ \ \ \ start\_conversion:}
\DoxyCodeLine{00394\ \ \ \ \ \ \ \ \ in\ =\ (ICONV\_CONST\ \textcolor{keywordtype}{char}\ *)model-\/>word\_str[i];}
\DoxyCodeLine{00395\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Yes,\ this\ assumes\ that\ we\ don't\ have\ any\ NUL\ bytes.\ */}}
\DoxyCodeLine{00396\ \ \ \ \ \ \ \ \ inleft\ =\ strlen(in);}
\DoxyCodeLine{00397\ \ \ \ \ \ \ \ \ out\ =\ outbuf;}
\DoxyCodeLine{00398\ \ \ \ \ \ \ \ \ outleft\ =\ maxlen;}
\DoxyCodeLine{00399\ }
\DoxyCodeLine{00400\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{while}\ ((result\ =\ iconv(ic,\ \&in,\ \&inleft,\ \&out,\ \&outleft))\ ==\ (\textcolor{keywordtype}{size\_t})-\/1)\ \{}
\DoxyCodeLine{00401\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (errno\ !=\ E2BIG)\ \{}
\DoxyCodeLine{00402\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ FIXME:\ if\ we\ already\ converted\ any\ words,\ then\ they}}
\DoxyCodeLine{00403\ \textcolor{comment}{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ *\ are\ going\ to\ be\ in\ an\ inconsistent\ state.\ */}}
\DoxyCodeLine{00404\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR\_SYSTEM(\textcolor{stringliteral}{"{}iconv()\ failed"{}});}
\DoxyCodeLine{00405\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_free(outbuf);}
\DoxyCodeLine{00406\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ hash\_table\_free(new\_wid);}
\DoxyCodeLine{00407\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00408\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00409\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Reset\ the\ internal\ state\ of\ conversion.\ */}}
\DoxyCodeLine{00410\ \ \ \ \ \ \ \ \ \ \ \ \ iconv(ic,\ NULL,\ NULL,\ NULL,\ NULL);}
\DoxyCodeLine{00411\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Make\ everything\ bigger.\ */}}
\DoxyCodeLine{00412\ \ \ \ \ \ \ \ \ \ \ \ \ maxlen\ *=\ 2;}
\DoxyCodeLine{00413\ \ \ \ \ \ \ \ \ \ \ \ \ out\ =\ outbuf\ =\ ckd\_realloc(outbuf,\ maxlen);}
\DoxyCodeLine{00414\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Reset\ the\ input\ pointers.\ */}}
\DoxyCodeLine{00415\ \ \ \ \ \ \ \ \ \ \ \ \ in\ =\ (ICONV\_CONST\ \textcolor{keywordtype}{char}\ *)model-\/>word\_str[i];}
\DoxyCodeLine{00416\ \ \ \ \ \ \ \ \ \ \ \ \ inleft\ =\ strlen(in);}
\DoxyCodeLine{00417\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00418\ }
\DoxyCodeLine{00419\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Now\ flush\ a\ shift-\/out\ sequence,\ if\ any.\ */}}
\DoxyCodeLine{00420\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ ((result\ =\ iconv(ic,\ NULL,\ NULL,\ \&out,\ \&outleft))\ ==\ (\textcolor{keywordtype}{size\_t})-\/1)\ \{}
\DoxyCodeLine{00421\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (errno\ !=\ E2BIG)\ \{}
\DoxyCodeLine{00422\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ FIXME:\ if\ we\ already\ converted\ any\ words,\ then\ they}}
\DoxyCodeLine{00423\ \textcolor{comment}{\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ *\ are\ going\ to\ be\ in\ an\ inconsistent\ state.\ */}}
\DoxyCodeLine{00424\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ E\_ERROR\_SYSTEM(\textcolor{stringliteral}{"{}iconv()\ failed\ (state\ reset\ sequence)"{}});}
\DoxyCodeLine{00425\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ckd\_free(outbuf);}
\DoxyCodeLine{00426\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ hash\_table\_free(new\_wid);}
\DoxyCodeLine{00427\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00428\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00429\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Reset\ the\ internal\ state\ of\ conversion.\ */}}
\DoxyCodeLine{00430\ \ \ \ \ \ \ \ \ \ \ \ \ iconv(ic,\ NULL,\ NULL,\ NULL,\ NULL);}
\DoxyCodeLine{00431\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Make\ everything\ bigger.\ */}}
\DoxyCodeLine{00432\ \ \ \ \ \ \ \ \ \ \ \ \ maxlen\ *=\ 2;}
\DoxyCodeLine{00433\ \ \ \ \ \ \ \ \ \ \ \ \ outbuf\ =\ ckd\_realloc(outbuf,\ maxlen);}
\DoxyCodeLine{00434\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Be\ very\ evil.\ */}}
\DoxyCodeLine{00435\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ start\_conversion;}
\DoxyCodeLine{00436\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00437\ }
\DoxyCodeLine{00438\ \ \ \ \ \ \ \ \ result\ =\ maxlen\ -\/\ outleft;}
\DoxyCodeLine{00439\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Okay,\ that\ was\ hard,\ now\ let's\ go\ shopping.\ */}}
\DoxyCodeLine{00440\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (writable)\ \{}
\DoxyCodeLine{00441\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Grow\ or\ shrink\ the\ output\ string\ as\ necessary.\ */}}
\DoxyCodeLine{00442\ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>word\_str[i]\ =\ ckd\_realloc(model-\/>word\_str[i],\ result\ +\ 1);}
\DoxyCodeLine{00443\ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>word\_str[i][result]\ =\ \textcolor{charliteral}{'\(\backslash\)0'};}
\DoxyCodeLine{00444\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00445\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00446\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ It\ actually\ was\ not\ allocated\ previously,\ so\ do\ that\ now.\ */}}
\DoxyCodeLine{00447\ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>word\_str[i]\ =\ ckd\_calloc(result\ +\ 1,\ 1);}
\DoxyCodeLine{00448\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00449\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Copy\ the\ new\ thing\ in.\ */}}
\DoxyCodeLine{00450\ \ \ \ \ \ \ \ \ memcpy(model-\/>word\_str[i],\ outbuf,\ result);}
\DoxyCodeLine{00451\ }
\DoxyCodeLine{00452\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Now\ update\ the\ hash\ table.\ \ We\ might\ have\ terrible}}
\DoxyCodeLine{00453\ \textcolor{comment}{\ \ \ \ \ \ \ \ \ *\ collisions\ if\ a\ non-\/reversible\ conversion\ was\ requested.,}}
\DoxyCodeLine{00454\ \textcolor{comment}{\ \ \ \ \ \ \ \ \ *\ so\ warn\ about\ them.\ */}}
\DoxyCodeLine{00455\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (hash\_table\_enter\_int32(new\_wid,\ model-\/>word\_str[i],\ i)\ !=\ i)\ \{}
\DoxyCodeLine{00456\ \ \ \ \ \ \ \ \ \ \ \ \ E\_WARN(\textcolor{stringliteral}{"{}Duplicate\ word\ in\ dictionary\ after\ conversion:\ \%s\(\backslash\)n"{}},}
\DoxyCodeLine{00457\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>word\_str[i]);}
\DoxyCodeLine{00458\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00459\ \ \ \ \ \}}
\DoxyCodeLine{00460\ \ \ \ \ ckd\_free(outbuf);}
\DoxyCodeLine{00461\ \ \ \ \ iconv\_close(ic);}
\DoxyCodeLine{00462\ \ \ \ \ \textcolor{comment}{/*\ Swap\ out\ the\ hash\ table.\ */}}
\DoxyCodeLine{00463\ \ \ \ \ hash\_table\_free(model-\/>wid);}
\DoxyCodeLine{00464\ \ \ \ \ model-\/>wid\ =\ new\_wid;}
\DoxyCodeLine{00465\ }
\DoxyCodeLine{00466\ \ \ \ \ \textcolor{keywordflow}{return}\ 0;}
\DoxyCodeLine{00467\ \}}
\DoxyCodeLine{00468\ \textcolor{preprocessor}{\#else\ }\textcolor{comment}{/*\ !HAVE\_ICONV\ */}\textcolor{preprocessor}{}}
\DoxyCodeLine{00469\ \textcolor{keywordtype}{int}}
\DoxyCodeLine{00470\ ngram\_model\_recode(ngram\_model\_t\ *model,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *from,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *to)}
\DoxyCodeLine{00471\ \{}
\DoxyCodeLine{00472\ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00473\ \}}
\DoxyCodeLine{00474\ \textcolor{preprocessor}{\#endif\ }\textcolor{comment}{/*\ !HAVE\_ICONV\ */}\textcolor{preprocessor}{}}
\DoxyCodeLine{00475\ }
\DoxyCodeLine{00476\ \textcolor{keywordtype}{int}}
\DoxyCodeLine{00477\ ngram\_model\_apply\_weights(ngram\_model\_t\ *model,}
\DoxyCodeLine{00478\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ float32\ lw,\ float32\ wip,\ float32\ uw)}
\DoxyCodeLine{00479\ \{}
\DoxyCodeLine{00480\ \ \ \ \ \textcolor{keywordflow}{return}\ (*model-\/>funcs-\/>apply\_weights)(model,\ lw,\ wip,\ uw);}
\DoxyCodeLine{00481\ \}}
\DoxyCodeLine{00482\ }
\DoxyCodeLine{00483\ float32}
\DoxyCodeLine{00484\ ngram\_model\_get\_weights(ngram\_model\_t\ *model,\ int32\ *out\_log\_wip,}
\DoxyCodeLine{00485\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ *out\_log\_uw)}
\DoxyCodeLine{00486\ \{}
\DoxyCodeLine{00487\ \ \ \ \ \textcolor{keywordflow}{if}\ (out\_log\_wip)\ *out\_log\_wip\ =\ model-\/>log\_wip;}
\DoxyCodeLine{00488\ \ \ \ \ \textcolor{keywordflow}{if}\ (out\_log\_uw)\ *out\_log\_uw\ =\ model-\/>log\_uw;}
\DoxyCodeLine{00489\ \ \ \ \ \textcolor{keywordflow}{return}\ model-\/>lw;}
\DoxyCodeLine{00490\ \}}
\DoxyCodeLine{00491\ }
\DoxyCodeLine{00492\ }
\DoxyCodeLine{00493\ int32}
\DoxyCodeLine{00494\ ngram\_ng\_score(ngram\_model\_t\ *model,\ int32\ wid,\ int32\ *history,}
\DoxyCodeLine{00495\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ n\_hist,\ int32\ *n\_used)}
\DoxyCodeLine{00496\ \{}
\DoxyCodeLine{00497\ \ \ \ \ int32\ score,\ class\_weight\ =\ 0;}
\DoxyCodeLine{00498\ \ \ \ \ \textcolor{keywordtype}{int}\ i;}
\DoxyCodeLine{00499\ }
\DoxyCodeLine{00500\ \ \ \ \ \textcolor{comment}{/*\ Closed\ vocabulary,\ OOV\ word\ probability\ is\ zero\ */}}
\DoxyCodeLine{00501\ \ \ \ \ \textcolor{keywordflow}{if}\ (wid\ ==\ NGRAM\_INVALID\_WID)}
\DoxyCodeLine{00502\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ model-\/>log\_zero;}
\DoxyCodeLine{00503\ }
\DoxyCodeLine{00504\ \ \ \ \ \textcolor{comment}{/*\ "{}Declassify"{}\ wid\ and\ history\ */}}
\DoxyCodeLine{00505\ \ \ \ \ \textcolor{keywordflow}{if}\ (NGRAM\_IS\_CLASSWID(wid))\ \{}
\DoxyCodeLine{00506\ \ \ \ \ \ \ \ \ ngram\_class\_t\ *lmclass\ =\ model-\/>classes[NGRAM\_CLASSID(wid)];}
\DoxyCodeLine{00507\ }
\DoxyCodeLine{00508\ \ \ \ \ \ \ \ \ class\_weight\ =\ ngram\_class\_prob(lmclass,\ wid);}
\DoxyCodeLine{00509\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (class\_weight\ ==\ 1)\ \textcolor{comment}{/*\ Meaning,\ not\ found\ in\ class.\ */}}
\DoxyCodeLine{00510\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ model-\/>log\_zero;}
\DoxyCodeLine{00511\ \ \ \ \ \ \ \ \ wid\ =\ lmclass-\/>tag\_wid;}
\DoxyCodeLine{00512\ \ \ \ \ \}}
\DoxyCodeLine{00513\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ n\_hist;\ ++i)\ \{}
\DoxyCodeLine{00514\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (history[i]\ !=\ NGRAM\_INVALID\_WID\ \&\&\ NGRAM\_IS\_CLASSWID(history[i]))}
\DoxyCodeLine{00515\ \ \ \ \ \ \ \ \ \ \ \ \ history[i]\ =\ model-\/>classes[NGRAM\_CLASSID(history[i])]-\/>tag\_wid;}
\DoxyCodeLine{00516\ \ \ \ \ \}}
\DoxyCodeLine{00517\ \ \ \ \ score\ =\ (*model-\/>funcs-\/>score)(model,\ wid,\ history,\ n\_hist,\ n\_used);}
\DoxyCodeLine{00518\ }
\DoxyCodeLine{00519\ \ \ \ \ \textcolor{comment}{/*\ Multiply\ by\ unigram\ in-\/class\ weight.\ */}}
\DoxyCodeLine{00520\ \ \ \ \ \textcolor{keywordflow}{return}\ score\ +\ class\_weight;}
\DoxyCodeLine{00521\ \}}
\DoxyCodeLine{00522\ }
\DoxyCodeLine{00523\ int32}
\DoxyCodeLine{00524\ ngram\_score(ngram\_model\_t\ *model,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *word,\ ...)}
\DoxyCodeLine{00525\ \{}
\DoxyCodeLine{00526\ \ \ \ \ va\_list\ history;}
\DoxyCodeLine{00527\ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *hword;}
\DoxyCodeLine{00528\ \ \ \ \ int32\ *histid;}
\DoxyCodeLine{00529\ \ \ \ \ int32\ n\_hist;}
\DoxyCodeLine{00530\ \ \ \ \ int32\ n\_used;}
\DoxyCodeLine{00531\ \ \ \ \ int32\ prob;}
\DoxyCodeLine{00532\ }
\DoxyCodeLine{00533\ \ \ \ \ va\_start(history,\ word);}
\DoxyCodeLine{00534\ \ \ \ \ n\_hist\ =\ 0;}
\DoxyCodeLine{00535\ \ \ \ \ \textcolor{keywordflow}{while}\ ((hword\ =\ va\_arg(history,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *))\ !=\ NULL)}
\DoxyCodeLine{00536\ \ \ \ \ \ \ \ \ ++n\_hist;}
\DoxyCodeLine{00537\ \ \ \ \ va\_end(history);}
\DoxyCodeLine{00538\ }
\DoxyCodeLine{00539\ \ \ \ \ histid\ =\ ckd\_calloc(n\_hist,\ \textcolor{keyword}{sizeof}(*histid));}
\DoxyCodeLine{00540\ \ \ \ \ va\_start(history,\ word);}
\DoxyCodeLine{00541\ \ \ \ \ n\_hist\ =\ 0;}
\DoxyCodeLine{00542\ \ \ \ \ \textcolor{keywordflow}{while}\ ((hword\ =\ va\_arg(history,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *))\ !=\ NULL)\ \{}
\DoxyCodeLine{00543\ \ \ \ \ \ \ \ \ histid[n\_hist]\ =\ ngram\_wid(model,\ hword);}
\DoxyCodeLine{00544\ \ \ \ \ \ \ \ \ ++n\_hist;}
\DoxyCodeLine{00545\ \ \ \ \ \}}
\DoxyCodeLine{00546\ \ \ \ \ va\_end(history);}
\DoxyCodeLine{00547\ }
\DoxyCodeLine{00548\ \ \ \ \ prob\ =\ ngram\_ng\_score(model,\ ngram\_wid(model,\ word),}
\DoxyCodeLine{00549\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ histid,\ n\_hist,\ \&n\_used);}
\DoxyCodeLine{00550\ \ \ \ \ ckd\_free(histid);}
\DoxyCodeLine{00551\ \ \ \ \ \textcolor{keywordflow}{return}\ prob;}
\DoxyCodeLine{00552\ \}}
\DoxyCodeLine{00553\ }
\DoxyCodeLine{00554\ int32}
\DoxyCodeLine{00555\ ngram\_tg\_score(ngram\_model\_t\ *model,\ int32\ w3,\ int32\ w2,\ int32\ w1,\ int32\ *n\_used)}
\DoxyCodeLine{00556\ \{}
\DoxyCodeLine{00557\ \ \ \ \ int32\ hist[2];}
\DoxyCodeLine{00558\ \ \ \ \ hist[0]\ =\ w2;}
\DoxyCodeLine{00559\ \ \ \ \ hist[1]\ =\ w1;}
\DoxyCodeLine{00560\ \ \ \ \ \textcolor{keywordflow}{return}\ ngram\_ng\_score(model,\ w3,\ hist,\ 2,\ n\_used);}
\DoxyCodeLine{00561\ \}}
\DoxyCodeLine{00562\ }
\DoxyCodeLine{00563\ int32}
\DoxyCodeLine{00564\ ngram\_bg\_score(ngram\_model\_t\ *model,\ int32\ w2,\ int32\ w1,\ int32\ *n\_used)}
\DoxyCodeLine{00565\ \{}
\DoxyCodeLine{00566\ \ \ \ \ \textcolor{keywordflow}{return}\ ngram\_ng\_score(model,\ w2,\ \&w1,\ 1,\ n\_used);}
\DoxyCodeLine{00567\ \}}
\DoxyCodeLine{00568\ }
\DoxyCodeLine{00569\ int32}
\DoxyCodeLine{00570\ ngram\_ng\_prob(ngram\_model\_t\ *model,\ int32\ wid,\ int32\ *history,}
\DoxyCodeLine{00571\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ n\_hist,\ int32\ *n\_used)}
\DoxyCodeLine{00572\ \{}
\DoxyCodeLine{00573\ \ \ \ \ int32\ prob,\ class\_weight\ =\ 0;}
\DoxyCodeLine{00574\ \ \ \ \ \textcolor{keywordtype}{int}\ i;}
\DoxyCodeLine{00575\ }
\DoxyCodeLine{00576\ \ \ \ \ \textcolor{comment}{/*\ Closed\ vocabulary,\ OOV\ word\ probability\ is\ zero\ */}}
\DoxyCodeLine{00577\ \ \ \ \ \textcolor{keywordflow}{if}\ (wid\ ==\ NGRAM\_INVALID\_WID)}
\DoxyCodeLine{00578\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ model-\/>log\_zero;}
\DoxyCodeLine{00579\ }
\DoxyCodeLine{00580\ \ \ \ \ \textcolor{comment}{/*\ "{}Declassify"{}\ wid\ and\ history\ */}}
\DoxyCodeLine{00581\ \ \ \ \ \textcolor{keywordflow}{if}\ (NGRAM\_IS\_CLASSWID(wid))\ \{}
\DoxyCodeLine{00582\ \ \ \ \ \ \ \ \ ngram\_class\_t\ *lmclass\ =\ model-\/>classes[NGRAM\_CLASSID(wid)];}
\DoxyCodeLine{00583\ }
\DoxyCodeLine{00584\ \ \ \ \ \ \ \ \ class\_weight\ =\ ngram\_class\_prob(lmclass,\ wid);}
\DoxyCodeLine{00585\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (class\_weight\ ==\ 1)\ \textcolor{comment}{/*\ Meaning,\ not\ found\ in\ class.\ */}}
\DoxyCodeLine{00586\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ class\_weight;}
\DoxyCodeLine{00587\ \ \ \ \ \ \ \ \ wid\ =\ lmclass-\/>tag\_wid;}
\DoxyCodeLine{00588\ \ \ \ \ \}}
\DoxyCodeLine{00589\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ n\_hist;\ ++i)\ \{}
\DoxyCodeLine{00590\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (history[i]\ !=\ NGRAM\_INVALID\_WID\ \&\&\ NGRAM\_IS\_CLASSWID(history[i]))}
\DoxyCodeLine{00591\ \ \ \ \ \ \ \ \ \ \ \ \ history[i]\ =\ model-\/>classes[NGRAM\_CLASSID(history[i])]-\/>tag\_wid;}
\DoxyCodeLine{00592\ \ \ \ \ \}}
\DoxyCodeLine{00593\ \ \ \ \ prob\ =\ (*model-\/>funcs-\/>raw\_score)(model,\ wid,\ history,}
\DoxyCodeLine{00594\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ n\_hist,\ n\_used);}
\DoxyCodeLine{00595\ \ \ \ \ \textcolor{comment}{/*\ Multiply\ by\ unigram\ in-\/class\ weight.\ */}}
\DoxyCodeLine{00596\ \ \ \ \ \textcolor{keywordflow}{return}\ prob\ +\ class\_weight;}
\DoxyCodeLine{00597\ \}}
\DoxyCodeLine{00598\ }
\DoxyCodeLine{00599\ int32}
\DoxyCodeLine{00600\ ngram\_prob(ngram\_model\_t\ *model,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *word,\ ...)}
\DoxyCodeLine{00601\ \{}
\DoxyCodeLine{00602\ \ \ \ \ va\_list\ history;}
\DoxyCodeLine{00603\ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *hword;}
\DoxyCodeLine{00604\ \ \ \ \ int32\ *histid;}
\DoxyCodeLine{00605\ \ \ \ \ int32\ n\_hist;}
\DoxyCodeLine{00606\ \ \ \ \ int32\ n\_used;}
\DoxyCodeLine{00607\ \ \ \ \ int32\ prob;}
\DoxyCodeLine{00608\ }
\DoxyCodeLine{00609\ \ \ \ \ va\_start(history,\ word);}
\DoxyCodeLine{00610\ \ \ \ \ n\_hist\ =\ 0;}
\DoxyCodeLine{00611\ \ \ \ \ \textcolor{keywordflow}{while}\ ((hword\ =\ va\_arg(history,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *))\ !=\ NULL)}
\DoxyCodeLine{00612\ \ \ \ \ \ \ \ \ ++n\_hist;}
\DoxyCodeLine{00613\ \ \ \ \ va\_end(history);}
\DoxyCodeLine{00614\ }
\DoxyCodeLine{00615\ \ \ \ \ histid\ =\ ckd\_calloc(n\_hist,\ \textcolor{keyword}{sizeof}(*histid));}
\DoxyCodeLine{00616\ \ \ \ \ va\_start(history,\ word);}
\DoxyCodeLine{00617\ \ \ \ \ n\_hist\ =\ 0;}
\DoxyCodeLine{00618\ \ \ \ \ \textcolor{keywordflow}{while}\ ((hword\ =\ va\_arg(history,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *))\ !=\ NULL)\ \{}
\DoxyCodeLine{00619\ \ \ \ \ \ \ \ \ histid[n\_hist]\ =\ ngram\_wid(model,\ hword);}
\DoxyCodeLine{00620\ \ \ \ \ \ \ \ \ ++n\_hist;}
\DoxyCodeLine{00621\ \ \ \ \ \}}
\DoxyCodeLine{00622\ \ \ \ \ va\_end(history);}
\DoxyCodeLine{00623\ }
\DoxyCodeLine{00624\ \ \ \ \ prob\ =\ ngram\_ng\_prob(model,\ ngram\_wid(model,\ word),}
\DoxyCodeLine{00625\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ histid,\ n\_hist,\ \&n\_used);}
\DoxyCodeLine{00626\ \ \ \ \ ckd\_free(histid);}
\DoxyCodeLine{00627\ \ \ \ \ \textcolor{keywordflow}{return}\ prob;}
\DoxyCodeLine{00628\ \}}
\DoxyCodeLine{00629\ }
\DoxyCodeLine{00630\ int32}
\DoxyCodeLine{00631\ ngram\_score\_to\_prob(ngram\_model\_t\ *base,\ int32\ score)}
\DoxyCodeLine{00632\ \{}
\DoxyCodeLine{00633\ \ \ \ \ int32\ prob;}
\DoxyCodeLine{00634\ }
\DoxyCodeLine{00635\ \ \ \ \ \textcolor{comment}{/*\ Undo\ insertion\ penalty.\ */}}
\DoxyCodeLine{00636\ \ \ \ \ prob\ =\ score\ -\/\ base-\/>log\_wip;}
\DoxyCodeLine{00637\ \ \ \ \ \textcolor{comment}{/*\ Undo\ language\ weight.\ */}}
\DoxyCodeLine{00638\ \ \ \ \ prob\ =\ (int32)(prob\ /\ base-\/>lw);}
\DoxyCodeLine{00639\ }
\DoxyCodeLine{00640\ \ \ \ \ \textcolor{keywordflow}{return}\ prob;}
\DoxyCodeLine{00641\ \}}
\DoxyCodeLine{00642\ }
\DoxyCodeLine{00643\ int32}
\DoxyCodeLine{00644\ ngram\_unknown\_wid(ngram\_model\_t\ *model)}
\DoxyCodeLine{00645\ \{}
\DoxyCodeLine{00646\ \ \ \ \ int32\ val;}
\DoxyCodeLine{00647\ }
\DoxyCodeLine{00648\ \ \ \ \ \textcolor{comment}{/*\ FIXME:\ This\ could\ be\ memoized\ for\ speed\ if\ necessary.\ */}}
\DoxyCodeLine{00649\ \ \ \ \ \textcolor{comment}{/*\ Look\ up\ <UNK>,\ if\ not\ found\ return\ NGRAM\_INVALID\_WID.\ */}}
\DoxyCodeLine{00650\ \ \ \ \ \textcolor{keywordflow}{if}\ (hash\_table\_lookup\_int32(model-\/>wid,\ \textcolor{stringliteral}{"{}<UNK>"{}},\ \&val)\ ==\ -\/1)}
\DoxyCodeLine{00651\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NGRAM\_INVALID\_WID;}
\DoxyCodeLine{00652\ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00653\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ val;}
\DoxyCodeLine{00654\ \}}
\DoxyCodeLine{00655\ }
\DoxyCodeLine{00656\ int32}
\DoxyCodeLine{00657\ ngram\_zero(ngram\_model\_t\ *model)}
\DoxyCodeLine{00658\ \{}
\DoxyCodeLine{00659\ \ \ \ \ \textcolor{keywordflow}{return}\ model-\/>log\_zero;}
\DoxyCodeLine{00660\ \}}
\DoxyCodeLine{00661\ }
\DoxyCodeLine{00662\ int32}
\DoxyCodeLine{00663\ ngram\_model\_get\_size(ngram\_model\_t\ *model)}
\DoxyCodeLine{00664\ \{}
\DoxyCodeLine{00665\ \ \ \textcolor{keywordflow}{if}\ (model\ !=\ NULL)}
\DoxyCodeLine{00666\ \ \ \ \ \textcolor{keywordflow}{return}\ model-\/>n;}
\DoxyCodeLine{00667\ \ \ \textcolor{keywordflow}{return}\ 0;}
\DoxyCodeLine{00668\ \}}
\DoxyCodeLine{00669\ }
\DoxyCodeLine{00670\ int32\ \textcolor{keyword}{const}\ *}
\DoxyCodeLine{00671\ ngram\_model\_get\_counts(ngram\_model\_t\ *model)}
\DoxyCodeLine{00672\ \{}
\DoxyCodeLine{00673\ \ \ \textcolor{keywordflow}{if}\ (model\ !=\ NULL)}
\DoxyCodeLine{00674\ \ \ \ \ \textcolor{keywordflow}{return}\ model-\/>n\_counts;}
\DoxyCodeLine{00675\ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00676\ \}}
\DoxyCodeLine{00677\ }
\DoxyCodeLine{00678\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00679\ ngram\_iter\_init(ngram\_iter\_t\ *itor,\ ngram\_model\_t\ *model,}
\DoxyCodeLine{00680\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{int}\ m,\ \textcolor{keywordtype}{int}\ successor)}
\DoxyCodeLine{00681\ \{}
\DoxyCodeLine{00682\ \ \ \ \ itor-\/>model\ =\ model;}
\DoxyCodeLine{00683\ \ \ \ \ itor-\/>wids\ =\ ckd\_calloc(model-\/>n,\ \textcolor{keyword}{sizeof}(*itor-\/>wids));}
\DoxyCodeLine{00684\ \ \ \ \ itor-\/>m\ =\ m;}
\DoxyCodeLine{00685\ \ \ \ \ itor-\/>successor\ =\ successor;}
\DoxyCodeLine{00686\ \}}
\DoxyCodeLine{00687\ }
\DoxyCodeLine{00688\ ngram\_iter\_t\ *}
\DoxyCodeLine{00689\ ngram\_model\_mgrams(ngram\_model\_t\ *model,\ \textcolor{keywordtype}{int}\ m)}
\DoxyCodeLine{00690\ \{}
\DoxyCodeLine{00691\ \ \ \ \ ngram\_iter\_t\ *itor;}
\DoxyCodeLine{00692\ \ \ \ \ \textcolor{comment}{/*\ The\ fact\ that\ m=n-\/1\ is\ not\ exactly\ obvious.\ \ Prevent\ accidents.\ */}}
\DoxyCodeLine{00693\ \ \ \ \ \textcolor{keywordflow}{if}\ (m\ >=\ model-\/>n)}
\DoxyCodeLine{00694\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00695\ \ \ \ \ \textcolor{keywordflow}{if}\ (model-\/>funcs-\/>mgrams\ ==\ NULL)}
\DoxyCodeLine{00696\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00697\ \ \ \ \ itor\ =\ (*model-\/>funcs-\/>mgrams)(model,\ m);}
\DoxyCodeLine{00698\ \ \ \ \ \textcolor{keywordflow}{return}\ itor;}
\DoxyCodeLine{00699\ \}}
\DoxyCodeLine{00700\ }
\DoxyCodeLine{00701\ ngram\_iter\_t\ *}
\DoxyCodeLine{00702\ ngram\_iter(ngram\_model\_t\ *model,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *word,\ ...)}
\DoxyCodeLine{00703\ \{}
\DoxyCodeLine{00704\ \ \ \ \ va\_list\ history;}
\DoxyCodeLine{00705\ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *hword;}
\DoxyCodeLine{00706\ \ \ \ \ int32\ *histid;}
\DoxyCodeLine{00707\ \ \ \ \ int32\ n\_hist;}
\DoxyCodeLine{00708\ \ \ \ \ ngram\_iter\_t\ *itor;}
\DoxyCodeLine{00709\ }
\DoxyCodeLine{00710\ \ \ \ \ va\_start(history,\ word);}
\DoxyCodeLine{00711\ \ \ \ \ n\_hist\ =\ 0;}
\DoxyCodeLine{00712\ \ \ \ \ \textcolor{keywordflow}{while}\ ((hword\ =\ va\_arg(history,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *))\ !=\ NULL)}
\DoxyCodeLine{00713\ \ \ \ \ \ \ \ \ ++n\_hist;}
\DoxyCodeLine{00714\ \ \ \ \ va\_end(history);}
\DoxyCodeLine{00715\ }
\DoxyCodeLine{00716\ \ \ \ \ histid\ =\ ckd\_calloc(n\_hist,\ \textcolor{keyword}{sizeof}(*histid));}
\DoxyCodeLine{00717\ \ \ \ \ va\_start(history,\ word);}
\DoxyCodeLine{00718\ \ \ \ \ n\_hist\ =\ 0;}
\DoxyCodeLine{00719\ \ \ \ \ \textcolor{keywordflow}{while}\ ((hword\ =\ va\_arg(history,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *))\ !=\ NULL)\ \{}
\DoxyCodeLine{00720\ \ \ \ \ \ \ \ \ histid[n\_hist]\ =\ ngram\_wid(model,\ hword);}
\DoxyCodeLine{00721\ \ \ \ \ \ \ \ \ ++n\_hist;}
\DoxyCodeLine{00722\ \ \ \ \ \}}
\DoxyCodeLine{00723\ \ \ \ \ va\_end(history);}
\DoxyCodeLine{00724\ }
\DoxyCodeLine{00725\ \ \ \ \ itor\ =\ ngram\_ng\_iter(model,\ ngram\_wid(model,\ word),\ histid,\ n\_hist);}
\DoxyCodeLine{00726\ \ \ \ \ ckd\_free(histid);}
\DoxyCodeLine{00727\ \ \ \ \ \textcolor{keywordflow}{return}\ itor;}
\DoxyCodeLine{00728\ \}}
\DoxyCodeLine{00729\ }
\DoxyCodeLine{00730\ ngram\_iter\_t\ *}
\DoxyCodeLine{00731\ ngram\_ng\_iter(ngram\_model\_t\ *model,\ int32\ wid,\ int32\ *history,\ int32\ n\_hist)}
\DoxyCodeLine{00732\ \{}
\DoxyCodeLine{00733\ \ \ \ \ \textcolor{keywordflow}{if}\ (n\_hist\ >=\ model-\/>n)}
\DoxyCodeLine{00734\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00735\ \ \ \ \ \textcolor{keywordflow}{if}\ (model-\/>funcs-\/>iter\ ==\ NULL)}
\DoxyCodeLine{00736\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00737\ \ \ \ \ \textcolor{keywordflow}{return}\ (*model-\/>funcs-\/>iter)(model,\ wid,\ history,\ n\_hist);}
\DoxyCodeLine{00738\ \}}
\DoxyCodeLine{00739\ }
\DoxyCodeLine{00740\ ngram\_iter\_t\ *}
\DoxyCodeLine{00741\ ngram\_iter\_successors(ngram\_iter\_t\ *itor)}
\DoxyCodeLine{00742\ \{}
\DoxyCodeLine{00743\ \ \ \ \ \textcolor{comment}{/*\ Stop\ when\ we\ are\ at\ the\ highest\ order\ N-\/Gram.\ */}}
\DoxyCodeLine{00744\ \ \ \ \ \textcolor{keywordflow}{if}\ (itor-\/>m\ ==\ itor-\/>model-\/>n\ -\/\ 1)}
\DoxyCodeLine{00745\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00746\ \ \ \ \ \textcolor{keywordflow}{return}\ (*itor-\/>model-\/>funcs-\/>successors)(itor);}
\DoxyCodeLine{00747\ \}}
\DoxyCodeLine{00748\ }
\DoxyCodeLine{00749\ int32\ \textcolor{keyword}{const}\ *}
\DoxyCodeLine{00750\ ngram\_iter\_get(ngram\_iter\_t\ *itor,}
\DoxyCodeLine{00751\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ *out\_score,}
\DoxyCodeLine{00752\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ *out\_bowt)}
\DoxyCodeLine{00753\ \{}
\DoxyCodeLine{00754\ \ \ \ \ \textcolor{keywordflow}{return}\ (*itor-\/>model-\/>funcs-\/>iter\_get)(itor,\ out\_score,\ out\_bowt);}
\DoxyCodeLine{00755\ \}}
\DoxyCodeLine{00756\ }
\DoxyCodeLine{00757\ ngram\_iter\_t\ *}
\DoxyCodeLine{00758\ ngram\_iter\_next(ngram\_iter\_t\ *itor)}
\DoxyCodeLine{00759\ \{}
\DoxyCodeLine{00760\ \ \ \ \ \textcolor{keywordflow}{return}\ (*itor-\/>model-\/>funcs-\/>iter\_next)(itor);}
\DoxyCodeLine{00761\ \}}
\DoxyCodeLine{00762\ }
\DoxyCodeLine{00763\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00764\ ngram\_iter\_free(ngram\_iter\_t\ *itor)}
\DoxyCodeLine{00765\ \{}
\DoxyCodeLine{00766\ \ \ \ \ ckd\_free(itor-\/>wids);}
\DoxyCodeLine{00767\ \ \ \ \ (*itor-\/>model-\/>funcs-\/>iter\_free)(itor);}
\DoxyCodeLine{00768\ \}}
\DoxyCodeLine{00769\ }
\DoxyCodeLine{00770\ int32}
\DoxyCodeLine{00771\ ngram\_wid(ngram\_model\_t\ *model,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *word)}
\DoxyCodeLine{00772\ \{}
\DoxyCodeLine{00773\ \ \ \ \ int32\ val;}
\DoxyCodeLine{00774\ }
\DoxyCodeLine{00775\ \ \ \ \ \textcolor{keywordflow}{if}\ (hash\_table\_lookup\_int32(model-\/>wid,\ word,\ \&val)\ ==\ -\/1)}
\DoxyCodeLine{00776\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ ngram\_unknown\_wid(model);}
\DoxyCodeLine{00777\ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{00778\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ val;}
\DoxyCodeLine{00779\ \}}
\DoxyCodeLine{00780\ }
\DoxyCodeLine{00781\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *}
\DoxyCodeLine{00782\ ngram\_word(ngram\_model\_t\ *model,\ int32\ wid)}
\DoxyCodeLine{00783\ \{}
\DoxyCodeLine{00784\ \ \ \ \ \textcolor{comment}{/*\ Remove\ any\ class\ tag\ */}}
\DoxyCodeLine{00785\ \ \ \ \ wid\ =\ NGRAM\_BASEWID(wid);}
\DoxyCodeLine{00786\ \ \ \ \ \textcolor{keywordflow}{if}\ (wid\ >=\ model-\/>n\_words)}
\DoxyCodeLine{00787\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NULL;}
\DoxyCodeLine{00788\ \ \ \ \ \textcolor{keywordflow}{return}\ model-\/>word\_str[wid];}
\DoxyCodeLine{00789\ \}}
\DoxyCodeLine{00790\ }
\DoxyCodeLine{00794\ int32}
\DoxyCodeLine{00795\ ngram\_add\_word\_internal(ngram\_model\_t\ *model,}
\DoxyCodeLine{00796\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *word,}
\DoxyCodeLine{00797\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ classid)}
\DoxyCodeLine{00798\ \{}
\DoxyCodeLine{00799\ \ \ \ \ \textcolor{keywordtype}{void}\ *dummy;}
\DoxyCodeLine{00800\ \ \ \ \ int32\ wid;}
\DoxyCodeLine{00801\ }
\DoxyCodeLine{00802\ \ \ \ \ \textcolor{comment}{/*\ Take\ the\ next\ available\ word\ ID\ */}}
\DoxyCodeLine{00803\ \ \ \ \ wid\ =\ model-\/>n\_words;}
\DoxyCodeLine{00804\ \ \ \ \ \textcolor{keywordflow}{if}\ (classid\ >=\ 0)\ \{}
\DoxyCodeLine{00805\ \ \ \ \ \ \ \ \ wid\ =\ NGRAM\_CLASSWID(wid,\ classid);}
\DoxyCodeLine{00806\ \ \ \ \ \}}
\DoxyCodeLine{00807\ \ \ \ \ \textcolor{comment}{/*\ Check\ for\ hash\ collisions.\ */}}
\DoxyCodeLine{00808\ \ \ \ \ \textcolor{keywordflow}{if}\ (hash\_table\_lookup(model-\/>wid,\ word,\ \&dummy)\ ==\ 0)\ \{}
\DoxyCodeLine{00809\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Duplicate\ definition\ of\ word\ \%s\(\backslash\)n"{}},\ word);}
\DoxyCodeLine{00810\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NGRAM\_INVALID\_WID;}
\DoxyCodeLine{00811\ \ \ \ \ \}}
\DoxyCodeLine{00812\ \ \ \ \ \textcolor{comment}{/*\ Reallocate\ word\_str\ if\ necessary.\ */}}
\DoxyCodeLine{00813\ \ \ \ \ \textcolor{keywordflow}{if}\ (model-\/>n\_words\ >=\ model-\/>n\_1g\_alloc)\ \{}
\DoxyCodeLine{00814\ \ \ \ \ \ \ \ \ model-\/>n\_1g\_alloc\ +=\ UG\_ALLOC\_STEP;}
\DoxyCodeLine{00815\ \ \ \ \ \ \ \ \ model-\/>word\_str\ =\ ckd\_realloc(model-\/>word\_str,}
\DoxyCodeLine{00816\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{sizeof}(*model-\/>word\_str)\ *\ model-\/>n\_1g\_alloc);}
\DoxyCodeLine{00817\ \ \ \ \ \}}
\DoxyCodeLine{00818\ \ \ \ \ \textcolor{comment}{/*\ Add\ the\ word\ string\ in\ the\ appropriate\ manner.\ */}}
\DoxyCodeLine{00819\ \ \ \ \ \textcolor{comment}{/*\ Class\ words\ are\ always\ dynamically\ allocated.\ */}}
\DoxyCodeLine{00820\ \ \ \ \ model-\/>word\_str[model-\/>n\_words]\ =\ ckd\_salloc(word);}
\DoxyCodeLine{00821\ \ \ \ \ \textcolor{comment}{/*\ Now\ enter\ it\ into\ the\ hash\ table.\ */}}
\DoxyCodeLine{00822\ \ \ \ \ \textcolor{keywordflow}{if}\ (hash\_table\_enter\_int32(model-\/>wid,\ model-\/>word\_str[model-\/>n\_words],\ wid)\ !=\ wid)\ \{}
\DoxyCodeLine{00823\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Hash\ insertion\ failed\ for\ word\ \%s\ =>\ \%p\ (should\ not\ happen)\(\backslash\)n"{}},}
\DoxyCodeLine{00824\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>word\_str[model-\/>n\_words],\ (\textcolor{keywordtype}{void}\ *)(\textcolor{keywordtype}{long})(wid));}
\DoxyCodeLine{00825\ \ \ \ \ \}}
\DoxyCodeLine{00826\ \ \ \ \ \textcolor{comment}{/*\ Increment\ number\ of\ words.\ */}}
\DoxyCodeLine{00827\ \ \ \ \ ++model-\/>n\_words;}
\DoxyCodeLine{00828\ \ \ \ \ \textcolor{keywordflow}{return}\ wid;}
\DoxyCodeLine{00829\ \}}
\DoxyCodeLine{00830\ }
\DoxyCodeLine{00831\ int32}
\DoxyCodeLine{00832\ ngram\_model\_add\_word(ngram\_model\_t\ *model,}
\DoxyCodeLine{00833\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *word,\ float32\ weight)}
\DoxyCodeLine{00834\ \{}
\DoxyCodeLine{00835\ \ \ \ \ int32\ wid,\ prob\ =\ model-\/>log\_zero;}
\DoxyCodeLine{00836\ }
\DoxyCodeLine{00837\ \ \ \ \ \textcolor{comment}{/*\ If\ we\ add\ word\ to\ unwritable\ model,\ we\ need\ to\ make\ it\ writable\ */}}
\DoxyCodeLine{00838\ \ \ \ \ \textcolor{keywordflow}{if}\ (!model-\/>writable)\ \{}
\DoxyCodeLine{00839\ \ \ \ \ \ \ \ \ E\_WARN(\textcolor{stringliteral}{"{}Can't\ add\ word\ '\%s'\ to\ read-\/only\ language\ model.\ "{}}}
\DoxyCodeLine{00840\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{stringliteral}{"{}Disable\ mmap\ with\ '-\/mmap\ no'\ to\ make\ it\ writable\(\backslash\)n"{}},\ word);}
\DoxyCodeLine{00841\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00842\ \ \ \ \ \}}
\DoxyCodeLine{00843\ }
\DoxyCodeLine{00844\ \ \ \ \ wid\ =\ ngram\_add\_word\_internal(model,\ word,\ -\/1);}
\DoxyCodeLine{00845\ \ \ \ \ \textcolor{keywordflow}{if}\ (wid\ ==\ NGRAM\_INVALID\_WID)}
\DoxyCodeLine{00846\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ wid;}
\DoxyCodeLine{00847\ }
\DoxyCodeLine{00848\ \ \ \ \ \textcolor{comment}{/*\ Do\ what\ needs\ to\ be\ done\ to\ add\ the\ word\ to\ the\ unigram.\ */}}
\DoxyCodeLine{00849\ \ \ \ \ \textcolor{keywordflow}{if}\ (model-\/>funcs\ \&\&\ model-\/>funcs-\/>add\_ug)}
\DoxyCodeLine{00850\ \ \ \ \ \ \ \ \ prob\ =\ (*model-\/>funcs-\/>add\_ug)(model,\ wid,\ logmath\_log(model-\/>lmath,\ weight));}
\DoxyCodeLine{00851\ \ \ \ \ \textcolor{keywordflow}{if}\ (prob\ ==\ 0)\ \{}
\DoxyCodeLine{00852\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{00853\ \ \ \ \ \}}
\DoxyCodeLine{00854\ \ \ \ \ \textcolor{keywordflow}{return}\ wid;}
\DoxyCodeLine{00855\ \}}
\DoxyCodeLine{00856\ }
\DoxyCodeLine{00857\ ngram\_class\_t\ *}
\DoxyCodeLine{00858\ ngram\_class\_new(ngram\_model\_t\ *model,\ int32\ tag\_wid,\ int32\ start\_wid,\ glist\_t\ classwords)}
\DoxyCodeLine{00859\ \{}
\DoxyCodeLine{00860\ \ \ \ \ ngram\_class\_t\ *lmclass;}
\DoxyCodeLine{00861\ \ \ \ \ gnode\_t\ *gn;}
\DoxyCodeLine{00862\ \ \ \ \ float32\ tprob;}
\DoxyCodeLine{00863\ \ \ \ \ \textcolor{keywordtype}{int}\ i;}
\DoxyCodeLine{00864\ }
\DoxyCodeLine{00865\ \ \ \ \ lmclass\ =\ ckd\_calloc(1,\ \textcolor{keyword}{sizeof}(*lmclass));}
\DoxyCodeLine{00866\ \ \ \ \ lmclass-\/>tag\_wid\ =\ tag\_wid;}
\DoxyCodeLine{00867\ \ \ \ \ \textcolor{comment}{/*\ wid\_base\ is\ the\ wid\ (minus\ class\ tag)\ of\ the\ first\ word\ in\ the\ list.\ */}}
\DoxyCodeLine{00868\ \ \ \ \ lmclass-\/>start\_wid\ =\ start\_wid;}
\DoxyCodeLine{00869\ \ \ \ \ lmclass-\/>n\_words\ =\ glist\_count(classwords);}
\DoxyCodeLine{00870\ \ \ \ \ lmclass-\/>prob1\ =\ ckd\_calloc(lmclass-\/>n\_words,\ \textcolor{keyword}{sizeof}(*lmclass-\/>prob1));}
\DoxyCodeLine{00871\ \ \ \ \ lmclass-\/>nword\_hash\ =\ NULL;}
\DoxyCodeLine{00872\ \ \ \ \ lmclass-\/>n\_hash\ =\ 0;}
\DoxyCodeLine{00873\ \ \ \ \ tprob\ =\ 0.0;}
\DoxyCodeLine{00874\ \ \ \ \ \textcolor{keywordflow}{for}\ (gn\ =\ classwords;\ gn;\ gn\ =\ gnode\_next(gn))\ \{}
\DoxyCodeLine{00875\ \ \ \ \ \ \ \ \ tprob\ +=\ gnode\_float32(gn);}
\DoxyCodeLine{00876\ \ \ \ \ \}}
\DoxyCodeLine{00877\ \ \ \ \ \textcolor{keywordflow}{if}\ (tprob\ >\ 1.1\ ||\ tprob\ <\ 0.9)\ \{}
\DoxyCodeLine{00878\ \ \ \ \ \ \ \ \ E\_WARN(\textcolor{stringliteral}{"{}Total\ class\ probability\ is\ \%f,\ will\ normalize\(\backslash\)n"{}},\ tprob);}
\DoxyCodeLine{00879\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (gn\ =\ classwords;\ gn;\ gn\ =\ gnode\_next(gn))\ \{}
\DoxyCodeLine{00880\ \ \ \ \ \ \ \ \ \ \ \ \ gn-\/>data.fl\ /=\ tprob;}
\DoxyCodeLine{00881\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00882\ \ \ \ \ \}}
\DoxyCodeLine{00883\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0,\ gn\ =\ classwords;\ gn;\ ++i,\ gn\ =\ gnode\_next(gn))\ \{}
\DoxyCodeLine{00884\ \ \ \ \ \ \ \ \ lmclass-\/>prob1[i]\ =\ logmath\_log(model-\/>lmath,\ gnode\_float32(gn));}
\DoxyCodeLine{00885\ \ \ \ \ \}}
\DoxyCodeLine{00886\ }
\DoxyCodeLine{00887\ \ \ \ \ \textcolor{keywordflow}{return}\ lmclass;}
\DoxyCodeLine{00888\ \}}
\DoxyCodeLine{00889\ }
\DoxyCodeLine{00890\ int32}
\DoxyCodeLine{00891\ ngram\_class\_add\_word(ngram\_class\_t\ *lmclass,\ int32\ wid,\ int32\ lweight)}
\DoxyCodeLine{00892\ \{}
\DoxyCodeLine{00893\ \ \ \ \ int32\ hash;}
\DoxyCodeLine{00894\ }
\DoxyCodeLine{00895\ \ \ \ \ \textcolor{keywordflow}{if}\ (lmclass-\/>nword\_hash\ ==\ NULL)\ \{}
\DoxyCodeLine{00896\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Initialize\ everything\ in\ it\ to\ -\/1\ */}}
\DoxyCodeLine{00897\ \ \ \ \ \ \ \ \ lmclass-\/>nword\_hash\ =\ ckd\_malloc(NGRAM\_HASH\_SIZE\ *\ \textcolor{keyword}{sizeof}(*lmclass-\/>nword\_hash));}
\DoxyCodeLine{00898\ \ \ \ \ \ \ \ \ memset(lmclass-\/>nword\_hash,\ 0xff,\ NGRAM\_HASH\_SIZE\ *\ \textcolor{keyword}{sizeof}(*lmclass-\/>nword\_hash));}
\DoxyCodeLine{00899\ \ \ \ \ \ \ \ \ lmclass-\/>n\_hash\ =\ NGRAM\_HASH\_SIZE;}
\DoxyCodeLine{00900\ \ \ \ \ \ \ \ \ lmclass-\/>n\_hash\_inuse\ =\ 0;}
\DoxyCodeLine{00901\ \ \ \ \ \}}
\DoxyCodeLine{00902\ \ \ \ \ \textcolor{comment}{/*\ Stupidest\ possible\ hash\ function.\ \ This\ will\ work\ pretty\ well}}
\DoxyCodeLine{00903\ \textcolor{comment}{\ \ \ \ \ *\ when\ this\ function\ is\ called\ repeatedly\ with\ contiguous\ word}}
\DoxyCodeLine{00904\ \textcolor{comment}{\ \ \ \ \ *\ IDs,\ though...\ */}}
\DoxyCodeLine{00905\ \ \ \ \ hash\ =\ wid\ \&\ (lmclass-\/>n\_hash\ -\/\ 1);}
\DoxyCodeLine{00906\ \ \ \ \ \textcolor{keywordflow}{if}\ (lmclass-\/>nword\_hash[hash].wid\ ==\ -\/1)\ \{}
\DoxyCodeLine{00907\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Good,\ no\ collision.\ */}}
\DoxyCodeLine{00908\ \ \ \ \ \ \ \ \ lmclass-\/>nword\_hash[hash].wid\ =\ wid;}
\DoxyCodeLine{00909\ \ \ \ \ \ \ \ \ lmclass-\/>nword\_hash[hash].prob1\ =\ lweight;}
\DoxyCodeLine{00910\ \ \ \ \ \ \ \ \ ++lmclass-\/>n\_hash\_inuse;}
\DoxyCodeLine{00911\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ hash;}
\DoxyCodeLine{00912\ \ \ \ \ \}}
\DoxyCodeLine{00913\ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00914\ \ \ \ \ \ \ \ \ int32\ next;\ }
\DoxyCodeLine{00915\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Collision...\ Find\ the\ end\ of\ the\ hash\ chain.\ */}}
\DoxyCodeLine{00916\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{while}\ (lmclass-\/>nword\_hash[hash].next\ !=\ -\/1)}
\DoxyCodeLine{00917\ \ \ \ \ \ \ \ \ \ \ \ \ hash\ =\ lmclass-\/>nword\_hash[hash].next;}
\DoxyCodeLine{00918\ \ \ \ \ \ \ \ \ assert(hash\ !=\ -\/1);}
\DoxyCodeLine{00919\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Does\ we\ has\ any\ more\ bukkit?\ */}}
\DoxyCodeLine{00920\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (lmclass-\/>n\_hash\_inuse\ ==\ lmclass-\/>n\_hash)\ \{}
\DoxyCodeLine{00921\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Oh\ noes!\ \ Ok,\ we\ makes\ more.\ */}}
\DoxyCodeLine{00922\ \ \ \ \ \ \ \ \ \ \ \ \ lmclass-\/>nword\_hash\ =\ ckd\_realloc(lmclass-\/>nword\_hash,\ }
\DoxyCodeLine{00923\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ lmclass-\/>n\_hash\ *\ 2\ *\ \textcolor{keyword}{sizeof}(*lmclass-\/>nword\_hash));}
\DoxyCodeLine{00924\ \ \ \ \ \ \ \ \ \ \ \ \ memset(lmclass-\/>nword\_hash\ +\ lmclass-\/>n\_hash,}
\DoxyCodeLine{00925\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ 0xff,\ lmclass-\/>n\_hash\ *\ \textcolor{keyword}{sizeof}(*lmclass-\/>nword\_hash));}
\DoxyCodeLine{00926\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Just\ use\ the\ next\ allocated\ one\ (easy)\ */}}
\DoxyCodeLine{00927\ \ \ \ \ \ \ \ \ \ \ \ \ next\ =\ lmclass-\/>n\_hash;}
\DoxyCodeLine{00928\ \ \ \ \ \ \ \ \ \ \ \ \ lmclass-\/>n\_hash\ *=\ 2;}
\DoxyCodeLine{00929\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00930\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{00931\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Look\ for\ any\ available\ bucket.\ \ We\ hope\ this\ doesn't\ happen.\ */}}
\DoxyCodeLine{00932\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (next\ =\ 0;\ next\ <\ lmclass-\/>n\_hash;\ ++next)}
\DoxyCodeLine{00933\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (lmclass-\/>nword\_hash[next].wid\ ==\ -\/1)}
\DoxyCodeLine{00934\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00935\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ This\ should\ absolutely\ not\ happen.\ */}}
\DoxyCodeLine{00936\ \ \ \ \ \ \ \ \ \ \ \ \ assert(next\ !=\ lmclass-\/>n\_hash);}
\DoxyCodeLine{00937\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{00938\ \ \ \ \ \ \ \ \ lmclass-\/>nword\_hash[next].wid\ =\ wid;}
\DoxyCodeLine{00939\ \ \ \ \ \ \ \ \ lmclass-\/>nword\_hash[next].prob1\ =\ lweight;}
\DoxyCodeLine{00940\ \ \ \ \ \ \ \ \ lmclass-\/>nword\_hash[hash].next\ =\ next;}
\DoxyCodeLine{00941\ \ \ \ \ \ \ \ \ ++lmclass-\/>n\_hash\_inuse;}
\DoxyCodeLine{00942\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ next;}
\DoxyCodeLine{00943\ \ \ \ \ \}}
\DoxyCodeLine{00944\ \}}
\DoxyCodeLine{00945\ }
\DoxyCodeLine{00946\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{00947\ ngram\_class\_free(ngram\_class\_t\ *lmclass)}
\DoxyCodeLine{00948\ \{}
\DoxyCodeLine{00949\ \ \ \ \ ckd\_free(lmclass-\/>nword\_hash);}
\DoxyCodeLine{00950\ \ \ \ \ ckd\_free(lmclass-\/>prob1);}
\DoxyCodeLine{00951\ \ \ \ \ ckd\_free(lmclass);}
\DoxyCodeLine{00952\ \}}
\DoxyCodeLine{00953\ }
\DoxyCodeLine{00954\ int32}
\DoxyCodeLine{00955\ ngram\_model\_add\_class\_word(ngram\_model\_t\ *model,}
\DoxyCodeLine{00956\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *classname,}
\DoxyCodeLine{00957\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *word,}
\DoxyCodeLine{00958\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ float32\ weight)}
\DoxyCodeLine{00959\ \{}
\DoxyCodeLine{00960\ \ \ \ \ ngram\_class\_t\ *lmclass;}
\DoxyCodeLine{00961\ \ \ \ \ int32\ classid,\ tag\_wid,\ wid,\ i,\ scale;}
\DoxyCodeLine{00962\ \ \ \ \ float32\ fprob;}
\DoxyCodeLine{00963\ }
\DoxyCodeLine{00964\ \ \ \ \ \textcolor{comment}{/*\ Find\ the\ class\ corresponding\ to\ classname.\ \ Linear\ search}}
\DoxyCodeLine{00965\ \textcolor{comment}{\ \ \ \ \ *\ probably\ okay\ here\ since\ there\ won't\ be\ very\ many\ classes,\ and}}
\DoxyCodeLine{00966\ \textcolor{comment}{\ \ \ \ \ *\ this\ doesn't\ have\ to\ be\ fast.\ */}}
\DoxyCodeLine{00967\ \ \ \ \ tag\_wid\ =\ ngram\_wid(model,\ classname);}
\DoxyCodeLine{00968\ \ \ \ \ \textcolor{keywordflow}{if}\ (tag\_wid\ ==\ NGRAM\_INVALID\_WID)\ \{}
\DoxyCodeLine{00969\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}No\ such\ word\ or\ class\ tag:\ \%s\(\backslash\)n"{}},\ classname);}
\DoxyCodeLine{00970\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ tag\_wid;}
\DoxyCodeLine{00971\ \ \ \ \ \}}
\DoxyCodeLine{00972\ \ \ \ \ \textcolor{keywordflow}{for}\ (classid\ =\ 0;\ classid\ <\ model-\/>n\_classes;\ ++classid)\ \{}
\DoxyCodeLine{00973\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (model-\/>classes[classid]-\/>tag\_wid\ ==\ tag\_wid)}
\DoxyCodeLine{00974\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{00975\ \ \ \ \ \}}
\DoxyCodeLine{00976\ \ \ \ \ \textcolor{comment}{/*\ Hmm,\ no\ such\ class.\ \ It's\ probably\ not\ a\ good\ idea\ to\ create\ one.\ */}}
\DoxyCodeLine{00977\ \ \ \ \ \textcolor{keywordflow}{if}\ (classid\ ==\ model-\/>n\_classes)\ \{}
\DoxyCodeLine{00978\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Word\ \%s\ is\ not\ a\ class\ tag\ (call\ ngram\_model\_add\_class()\ first)\(\backslash\)n"{}},\ classname);}
\DoxyCodeLine{00979\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ NGRAM\_INVALID\_WID;}
\DoxyCodeLine{00980\ \ \ \ \ \}}
\DoxyCodeLine{00981\ \ \ \ \ lmclass\ =\ model-\/>classes[classid];}
\DoxyCodeLine{00982\ }
\DoxyCodeLine{00983\ \ \ \ \ \textcolor{comment}{/*\ Add\ this\ word\ to\ the\ model's\ set\ of\ words.\ */}}
\DoxyCodeLine{00984\ \ \ \ \ wid\ =\ ngram\_add\_word\_internal(model,\ word,\ classid);}
\DoxyCodeLine{00985\ \ \ \ \ \textcolor{keywordflow}{if}\ (wid\ ==\ NGRAM\_INVALID\_WID)}
\DoxyCodeLine{00986\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ wid;}
\DoxyCodeLine{00987\ }
\DoxyCodeLine{00988\ \ \ \ \ \textcolor{comment}{/*\ This\ is\ the\ fixed\ probability\ of\ the\ new\ word.\ */}}
\DoxyCodeLine{00989\ \ \ \ \ fprob\ =\ weight\ *\ 1.0f\ /\ (lmclass-\/>n\_words\ +\ lmclass-\/>n\_hash\_inuse\ +\ 1);}
\DoxyCodeLine{00990\ \ \ \ \ \textcolor{comment}{/*\ Now\ normalize\ everything\ else\ to\ fit\ it\ in.\ \ This\ is}}
\DoxyCodeLine{00991\ \textcolor{comment}{\ \ \ \ \ *\ accomplished\ by\ simply\ scaling\ all\ the\ other\ probabilities}}
\DoxyCodeLine{00992\ \textcolor{comment}{\ \ \ \ \ *\ by\ (1-\/fprob).\ */}}
\DoxyCodeLine{00993\ \ \ \ \ scale\ =\ logmath\_log(model-\/>lmath,\ 1.0\ -\/\ fprob);}
\DoxyCodeLine{00994\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ lmclass-\/>n\_words;\ ++i)}
\DoxyCodeLine{00995\ \ \ \ \ \ \ \ \ lmclass-\/>prob1[i]\ +=\ scale;}
\DoxyCodeLine{00996\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ lmclass-\/>n\_hash;\ ++i)}
\DoxyCodeLine{00997\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (lmclass-\/>nword\_hash[i].wid\ !=\ -\/1)}
\DoxyCodeLine{00998\ \ \ \ \ \ \ \ \ \ \ \ \ lmclass-\/>nword\_hash[i].prob1\ +=\ scale;}
\DoxyCodeLine{00999\ }
\DoxyCodeLine{01000\ \ \ \ \ \textcolor{comment}{/*\ Now\ add\ it\ to\ the\ class\ hash\ table.\ */}}
\DoxyCodeLine{01001\ \ \ \ \ \textcolor{keywordflow}{return}\ ngram\_class\_add\_word(lmclass,\ wid,\ logmath\_log(model-\/>lmath,\ fprob));}
\DoxyCodeLine{01002\ \}}
\DoxyCodeLine{01003\ }
\DoxyCodeLine{01004\ int32}
\DoxyCodeLine{01005\ ngram\_model\_add\_class(ngram\_model\_t\ *model,}
\DoxyCodeLine{01006\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *classname,}
\DoxyCodeLine{01007\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ float32\ classweight,}
\DoxyCodeLine{01008\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{char}\ **words,}
\DoxyCodeLine{01009\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ float32\ *weights,}
\DoxyCodeLine{01010\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ n\_words)}
\DoxyCodeLine{01011\ \{}
\DoxyCodeLine{01012\ \ \ \ \ ngram\_class\_t\ *lmclass;}
\DoxyCodeLine{01013\ \ \ \ \ glist\_t\ classwords\ =\ NULL;}
\DoxyCodeLine{01014\ \ \ \ \ int32\ i,\ start\_wid\ =\ -\/1;}
\DoxyCodeLine{01015\ \ \ \ \ int32\ classid,\ tag\_wid;}
\DoxyCodeLine{01016\ }
\DoxyCodeLine{01017\ \ \ \ \ \textcolor{comment}{/*\ Check\ if\ classname\ already\ exists\ in\ model.\ \ If\ not,\ add\ it.*/}}
\DoxyCodeLine{01018\ \ \ \ \ \textcolor{keywordflow}{if}\ ((tag\_wid\ =\ ngram\_wid(model,\ classname))\ ==\ ngram\_unknown\_wid(model))\ \{}
\DoxyCodeLine{01019\ \ \ \ \ \ \ \ \ tag\_wid\ =\ ngram\_model\_add\_word(model,\ classname,\ classweight);}
\DoxyCodeLine{01020\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (tag\_wid\ ==\ NGRAM\_INVALID\_WID)}
\DoxyCodeLine{01021\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{01022\ \ \ \ \ \}}
\DoxyCodeLine{01023\ }
\DoxyCodeLine{01024\ \ \ \ \ \textcolor{keywordflow}{if}\ (model-\/>n\_classes\ ==\ 128)\ \{}
\DoxyCodeLine{01025\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}Number\ of\ classes\ cannot\ exceed\ 128\ (sorry)\(\backslash\)n"{}});}
\DoxyCodeLine{01026\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{01027\ \ \ \ \ \}}
\DoxyCodeLine{01028\ \ \ \ \ classid\ =\ model-\/>n\_classes;}
\DoxyCodeLine{01029\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ n\_words;\ ++i)\ \{}
\DoxyCodeLine{01030\ \ \ \ \ \ \ \ \ int32\ wid;}
\DoxyCodeLine{01031\ }
\DoxyCodeLine{01032\ \ \ \ \ \ \ \ \ wid\ =\ ngram\_add\_word\_internal(model,\ words[i],\ classid);}
\DoxyCodeLine{01033\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (wid\ ==\ NGRAM\_INVALID\_WID)}
\DoxyCodeLine{01034\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{01035\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (start\_wid\ ==\ -\/1)}
\DoxyCodeLine{01036\ \ \ \ \ \ \ \ \ \ \ \ \ start\_wid\ =\ NGRAM\_BASEWID(wid);}
\DoxyCodeLine{01037\ \ \ \ \ \ \ \ \ classwords\ =\ glist\_add\_float32(classwords,\ weights[i]);}
\DoxyCodeLine{01038\ \ \ \ \ \}}
\DoxyCodeLine{01039\ \ \ \ \ classwords\ =\ glist\_reverse(classwords);}
\DoxyCodeLine{01040\ \ \ \ \ lmclass\ =\ ngram\_class\_new(model,\ tag\_wid,\ start\_wid,\ classwords);}
\DoxyCodeLine{01041\ \ \ \ \ glist\_free(classwords);}
\DoxyCodeLine{01042\ \ \ \ \ \textcolor{keywordflow}{if}\ (lmclass\ ==\ NULL)}
\DoxyCodeLine{01043\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{01044\ }
\DoxyCodeLine{01045\ \ \ \ \ ++model-\/>n\_classes;}
\DoxyCodeLine{01046\ \ \ \ \ \textcolor{keywordflow}{if}\ (model-\/>classes\ ==\ NULL)}
\DoxyCodeLine{01047\ \ \ \ \ \ \ \ \ model-\/>classes\ =\ ckd\_calloc(1,\ \textcolor{keyword}{sizeof}(*model-\/>classes));}
\DoxyCodeLine{01048\ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{01049\ \ \ \ \ \ \ \ \ model-\/>classes\ =\ ckd\_realloc(model-\/>classes,}
\DoxyCodeLine{01050\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ model-\/>n\_classes\ *\ \textcolor{keyword}{sizeof}(*model-\/>classes));}
\DoxyCodeLine{01051\ \ \ \ \ model-\/>classes[classid]\ =\ lmclass;}
\DoxyCodeLine{01052\ \ \ \ \ \textcolor{keywordflow}{return}\ classid;}
\DoxyCodeLine{01053\ \}}
\DoxyCodeLine{01054\ }
\DoxyCodeLine{01055\ int32}
\DoxyCodeLine{01056\ ngram\_class\_prob(ngram\_class\_t\ *lmclass,\ int32\ wid)}
\DoxyCodeLine{01057\ \{}
\DoxyCodeLine{01058\ \ \ \ \ int32\ base\_wid\ =\ NGRAM\_BASEWID(wid);}
\DoxyCodeLine{01059\ }
\DoxyCodeLine{01060\ \ \ \ \ \textcolor{keywordflow}{if}\ (base\_wid\ <\ lmclass-\/>start\_wid}
\DoxyCodeLine{01061\ \ \ \ \ \ \ \ \ ||\ base\_wid\ >\ lmclass-\/>start\_wid\ +\ lmclass-\/>n\_words)\ \{}
\DoxyCodeLine{01062\ \ \ \ \ \ \ \ \ int32\ hash;}
\DoxyCodeLine{01063\ }
\DoxyCodeLine{01064\ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Look\ it\ up\ in\ the\ hash\ table.\ */}}
\DoxyCodeLine{01065\ \ \ \ \ \ \ \ \ hash\ =\ wid\ \&\ (lmclass-\/>n\_hash\ -\/\ 1);}
\DoxyCodeLine{01066\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{while}\ (hash\ !=\ -\/1\ \&\&\ lmclass-\/>nword\_hash[hash].wid\ !=\ wid)}
\DoxyCodeLine{01067\ \ \ \ \ \ \ \ \ \ \ \ \ hash\ =\ lmclass-\/>nword\_hash[hash].next;}
\DoxyCodeLine{01068\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (hash\ ==\ -\/1)}
\DoxyCodeLine{01069\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ 1;}
\DoxyCodeLine{01070\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ lmclass-\/>nword\_hash[hash].prob1;}
\DoxyCodeLine{01071\ \ \ \ \ \}}
\DoxyCodeLine{01072\ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{01073\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ lmclass-\/>prob1[base\_wid\ -\/\ lmclass-\/>start\_wid];}
\DoxyCodeLine{01074\ \ \ \ \ \}}
\DoxyCodeLine{01075\ \}}
\DoxyCodeLine{01076\ }
\DoxyCodeLine{01077\ int32}
\DoxyCodeLine{01078\ read\_classdef\_file(hash\_table\_t\ *classes,\ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *file\_name)}
\DoxyCodeLine{01079\ \{}
\DoxyCodeLine{01080\ \ \ \ \ FILE\ *fp;}
\DoxyCodeLine{01081\ \ \ \ \ int32\ is\_pipe;}
\DoxyCodeLine{01082\ \ \ \ \ \textcolor{keywordtype}{int}\ inclass;\ \ }
\DoxyCodeLine{01083\ \ \ \ \ int32\ rv\ =\ -\/1;}
\DoxyCodeLine{01084\ \ \ \ \ gnode\_t\ *gn;}
\DoxyCodeLine{01085\ \ \ \ \ glist\_t\ classwords\ =\ NULL;}
\DoxyCodeLine{01086\ \ \ \ \ glist\_t\ classprobs\ =\ NULL;}
\DoxyCodeLine{01087\ \ \ \ \ \textcolor{keywordtype}{char}\ *classname\ =\ NULL;}
\DoxyCodeLine{01088\ }
\DoxyCodeLine{01089\ \ \ \ \ \textcolor{keywordflow}{if}\ ((fp\ =\ fopen\_comp(file\_name,\ \textcolor{stringliteral}{"{}r"{}},\ \&is\_pipe))\ ==\ NULL)\ \{}
\DoxyCodeLine{01090\ \ \ \ \ \ \ \ \ E\_ERROR(\textcolor{stringliteral}{"{}File\ \%s\ not\ found\(\backslash\)n"{}},\ file\_name);}
\DoxyCodeLine{01091\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{01092\ \ \ \ \ \}}
\DoxyCodeLine{01093\ }
\DoxyCodeLine{01094\ \ \ \ \ inclass\ =\ FALSE;}
\DoxyCodeLine{01095\ \ \ \ \ \textcolor{keywordflow}{while}\ (!feof(fp))\ \{}
\DoxyCodeLine{01096\ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{char}\ line[512];}
\DoxyCodeLine{01097\ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{char}\ *wptr[2];}
\DoxyCodeLine{01098\ \ \ \ \ \ \ \ \ \textcolor{keywordtype}{int}\ n\_words;}
\DoxyCodeLine{01099\ }
\DoxyCodeLine{01100\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (fgets(line,\ \textcolor{keyword}{sizeof}(line),\ fp)\ ==\ NULL)}
\DoxyCodeLine{01101\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{break};}
\DoxyCodeLine{01102\ }
\DoxyCodeLine{01103\ \ \ \ \ \ \ \ \ n\_words\ =\ str2words(line,\ wptr,\ 2);}
\DoxyCodeLine{01104\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (n\_words\ <=\ 0)}
\DoxyCodeLine{01105\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{continue};}
\DoxyCodeLine{01106\ }
\DoxyCodeLine{01107\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (inclass)\ \{}
\DoxyCodeLine{01108\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Look\ for\ an\ end\ of\ class\ marker.\ */}}
\DoxyCodeLine{01109\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (n\_words\ ==\ 2\ \&\&\ 0\ ==\ strcmp(wptr[0],\ \textcolor{stringliteral}{"{}END"{}}))\ \{}
\DoxyCodeLine{01110\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classdef\_t\ *classdef;}
\DoxyCodeLine{01111\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ gnode\_t\ *word,\ *weight;}
\DoxyCodeLine{01112\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ int32\ i;}
\DoxyCodeLine{01113\ }
\DoxyCodeLine{01114\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (classname\ ==\ NULL\ ||\ 0\ !=\ strcmp(wptr[1],\ classname))}
\DoxyCodeLine{01115\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{01116\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ inclass\ =\ FALSE;}
\DoxyCodeLine{01117\ }
\DoxyCodeLine{01118\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Construct\ a\ class\ from\ the\ list\ of\ words\ collected.\ */}}
\DoxyCodeLine{01119\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classdef\ =\ ckd\_calloc(1,\ \textcolor{keyword}{sizeof}(*classdef));}
\DoxyCodeLine{01120\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classwords\ =\ glist\_reverse(classwords);}
\DoxyCodeLine{01121\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classprobs\ =\ glist\_reverse(classprobs);}
\DoxyCodeLine{01122\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classdef-\/>n\_words\ =\ glist\_count(classwords);}
\DoxyCodeLine{01123\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classdef-\/>words\ =\ ckd\_calloc(classdef-\/>n\_words,}
\DoxyCodeLine{01124\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{sizeof}(*classdef-\/>words));}
\DoxyCodeLine{01125\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classdef-\/>weights\ =\ ckd\_calloc(classdef-\/>n\_words,}
\DoxyCodeLine{01126\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{sizeof}(*classdef-\/>weights));}
\DoxyCodeLine{01127\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ word\ =\ classwords;}
\DoxyCodeLine{01128\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ weight\ =\ classprobs;}
\DoxyCodeLine{01129\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ classdef-\/>n\_words;\ ++i)\ \{}
\DoxyCodeLine{01130\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classdef-\/>words[i]\ =\ gnode\_ptr(word);}
\DoxyCodeLine{01131\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classdef-\/>weights[i]\ =\ gnode\_float32(weight);}
\DoxyCodeLine{01132\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ word\ =\ gnode\_next(word);}
\DoxyCodeLine{01133\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ weight\ =\ gnode\_next(weight);}
\DoxyCodeLine{01134\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{01135\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ }
\DoxyCodeLine{01136\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Add\ this\ class\ to\ the\ hash\ table.\ */}}
\DoxyCodeLine{01137\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (hash\_table\_enter(classes,\ classname,\ classdef)\ !=\ classdef)\ \{}
\DoxyCodeLine{01138\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classdef\_free(classdef);}
\DoxyCodeLine{01139\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{01140\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{01141\ }
\DoxyCodeLine{01142\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Reset\ everything.\ */}}
\DoxyCodeLine{01143\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ glist\_free(classwords);}
\DoxyCodeLine{01144\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ glist\_free(classprobs);}
\DoxyCodeLine{01145\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classwords\ =\ NULL;}
\DoxyCodeLine{01146\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classprobs\ =\ NULL;}
\DoxyCodeLine{01147\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classname\ =\ NULL;}
\DoxyCodeLine{01148\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{01149\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{01150\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ float32\ fprob;}
\DoxyCodeLine{01151\ }
\DoxyCodeLine{01152\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (n\_words\ ==\ 2)}
\DoxyCodeLine{01153\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ fprob\ =\ (float32)atof\_c(wptr[1]);}
\DoxyCodeLine{01154\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}}
\DoxyCodeLine{01155\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ fprob\ =\ 1.0f;}
\DoxyCodeLine{01156\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Add\ it\ to\ the\ list\ of\ words\ for\ this\ class.\ */}}
\DoxyCodeLine{01157\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classwords\ =\ glist\_add\_ptr(classwords,\ ckd\_salloc(wptr[0]));}
\DoxyCodeLine{01158\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classprobs\ =\ glist\_add\_float32(classprobs,\ fprob);}
\DoxyCodeLine{01159\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{01160\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{01161\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{else}\ \{}
\DoxyCodeLine{01162\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Start\ a\ new\ LM\ class\ if\ the\ LMCLASS\ marker\ is\ seen\ */}}
\DoxyCodeLine{01163\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (n\_words\ ==\ 2\ \&\&\ 0\ ==\ strcmp(wptr[0],\ \textcolor{stringliteral}{"{}LMCLASS"{}}))\ \{}
\DoxyCodeLine{01164\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (inclass)}
\DoxyCodeLine{01165\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{01166\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ inclass\ =\ TRUE;}
\DoxyCodeLine{01167\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classname\ =\ ckd\_salloc(wptr[1]);}
\DoxyCodeLine{01168\ \ \ \ \ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{01169\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{comment}{/*\ Otherwise,\ just\ ignore\ whatever\ junk\ we\ got\ */}}
\DoxyCodeLine{01170\ \ \ \ \ \ \ \ \ \}}
\DoxyCodeLine{01171\ \ \ \ \ \}}
\DoxyCodeLine{01172\ \ \ \ \ rv\ =\ 0;\ \textcolor{comment}{/*\ Success.\ */}}
\DoxyCodeLine{01173\ }
\DoxyCodeLine{01174\ error\_out:}
\DoxyCodeLine{01175\ \ \ \ \ \textcolor{comment}{/*\ Free\ all\ the\ stuff\ we\ might\ have\ allocated.\ */}}
\DoxyCodeLine{01176\ \ \ \ \ fclose\_comp(fp,\ is\_pipe);}
\DoxyCodeLine{01177\ \ \ \ \ \textcolor{keywordflow}{for}\ (gn\ =\ classwords;\ gn;\ gn\ =\ gnode\_next(gn))}
\DoxyCodeLine{01178\ \ \ \ \ \ \ \ \ ckd\_free(gnode\_ptr(gn));}
\DoxyCodeLine{01179\ \ \ \ \ glist\_free(classwords);}
\DoxyCodeLine{01180\ \ \ \ \ glist\_free(classprobs);}
\DoxyCodeLine{01181\ \ \ \ \ ckd\_free(classname);}
\DoxyCodeLine{01182\ }
\DoxyCodeLine{01183\ \ \ \ \ \textcolor{keywordflow}{return}\ rv;}
\DoxyCodeLine{01184\ \}}
\DoxyCodeLine{01185\ }
\DoxyCodeLine{01186\ \textcolor{keywordtype}{void}}
\DoxyCodeLine{01187\ classdef\_free(classdef\_t\ *classdef)}
\DoxyCodeLine{01188\ \{}
\DoxyCodeLine{01189\ \ \ \ \ int32\ i;}
\DoxyCodeLine{01190\ \ \ \ \ \textcolor{keywordflow}{for}\ (i\ =\ 0;\ i\ <\ classdef-\/>n\_words;\ ++i)}
\DoxyCodeLine{01191\ \ \ \ \ \ \ \ \ ckd\_free(classdef-\/>words[i]);}
\DoxyCodeLine{01192\ \ \ \ \ ckd\_free(classdef-\/>words);}
\DoxyCodeLine{01193\ \ \ \ \ ckd\_free(classdef-\/>weights);}
\DoxyCodeLine{01194\ \ \ \ \ ckd\_free(classdef);}
\DoxyCodeLine{01195\ \}}
\DoxyCodeLine{01196\ }
\DoxyCodeLine{01197\ }
\DoxyCodeLine{01198\ int32}
\DoxyCodeLine{01199\ ngram\_model\_read\_classdef(ngram\_model\_t\ *model,}
\DoxyCodeLine{01200\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keyword}{const}\ \textcolor{keywordtype}{char}\ *file\_name)}
\DoxyCodeLine{01201\ \{}
\DoxyCodeLine{01202\ \ \ \ \ hash\_table\_t\ *classes;}
\DoxyCodeLine{01203\ \ \ \ \ glist\_t\ hl\ =\ NULL;}
\DoxyCodeLine{01204\ \ \ \ \ gnode\_t\ *gn;}
\DoxyCodeLine{01205\ \ \ \ \ int32\ rv\ =\ -\/1;}
\DoxyCodeLine{01206\ }
\DoxyCodeLine{01207\ \ \ \ \ classes\ =\ hash\_table\_new(0,\ FALSE);}
\DoxyCodeLine{01208\ \ \ \ \ \textcolor{keywordflow}{if}\ (read\_classdef\_file(classes,\ file\_name)\ <\ 0)\ \{}
\DoxyCodeLine{01209\ \ \ \ \ \ \ \ \ hash\_table\_free(classes);}
\DoxyCodeLine{01210\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{return}\ -\/1;}
\DoxyCodeLine{01211\ \ \ \ \ \}}
\DoxyCodeLine{01212\ \ \ \ \ }
\DoxyCodeLine{01213\ \ \ \ \ \textcolor{comment}{/*\ Create\ a\ new\ class\ in\ the\ language\ model\ for\ each\ classdef.\ */}}
\DoxyCodeLine{01214\ \ \ \ \ hl\ =\ hash\_table\_tolist(classes,\ NULL);}
\DoxyCodeLine{01215\ \ \ \ \ \textcolor{keywordflow}{for}\ (gn\ =\ hl;\ gn;\ gn\ =\ gnode\_next(gn))\ \{}
\DoxyCodeLine{01216\ \ \ \ \ \ \ \ \ hash\_entry\_t\ *he\ =\ gnode\_ptr(gn);}
\DoxyCodeLine{01217\ \ \ \ \ \ \ \ \ classdef\_t\ *classdef\ =\ he-\/>val;}
\DoxyCodeLine{01218\ }
\DoxyCodeLine{01219\ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{if}\ (ngram\_model\_add\_class(model,\ he-\/>key,\ 1.0,}
\DoxyCodeLine{01220\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classdef-\/>words,}
\DoxyCodeLine{01221\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classdef-\/>weights,}
\DoxyCodeLine{01222\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ classdef-\/>n\_words)\ <\ 0)}
\DoxyCodeLine{01223\ \ \ \ \ \ \ \ \ \ \ \ \ \textcolor{keywordflow}{goto}\ error\_out;}
\DoxyCodeLine{01224\ \ \ \ \ \}}
\DoxyCodeLine{01225\ \ \ \ \ rv\ =\ 0;}
\DoxyCodeLine{01226\ }
\DoxyCodeLine{01227\ error\_out:}
\DoxyCodeLine{01228\ \ \ \ \ \textcolor{keywordflow}{for}\ (gn\ =\ hl;\ gn;\ gn\ =\ gnode\_next(gn))\ \{}
\DoxyCodeLine{01229\ \ \ \ \ \ \ \ \ hash\_entry\_t\ *he\ =\ gnode\_ptr(gn);}
\DoxyCodeLine{01230\ \ \ \ \ \ \ \ \ ckd\_free((\textcolor{keywordtype}{char}\ *)he-\/>key);}
\DoxyCodeLine{01231\ \ \ \ \ \ \ \ \ classdef\_free(he-\/>val);}
\DoxyCodeLine{01232\ \ \ \ \ \}}
\DoxyCodeLine{01233\ \ \ \ \ glist\_free(hl);}
\DoxyCodeLine{01234\ \ \ \ \ hash\_table\_free(classes);}
\DoxyCodeLine{01235\ \ \ \ \ \textcolor{keywordflow}{return}\ rv;}
\DoxyCodeLine{01236\ \}}

\end{DoxyCode}
