Skip to content

Commit

Permalink
Remove vestigial len parameter from language model calls
Browse files Browse the repository at this point in the history 
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@3856 1f5c12ca-751b-0410-a591-d2e778427230
  • Loading branch information
heafield committed Jan 27, 2011
1 parent d20c520 commit b3fa5e9
Show file tree
Hide file tree
Showing 20 changed files with 34 additions and 62 deletions.
5 changes: 2 additions & 3 deletions moses/src/LanguageModel.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,9 +222,8 @@ FFState* LanguageModel::Evaluate(
contextFactor[index++] = &m_implementation->GetSentenceStartArray();
}
}
unsigned int statelen;
FFState *res = m_implementation->NewState(ps);
float lmScore = ps ? m_implementation->GetValueGivenState(contextFactor, *res, &statelen) : m_implementation->GetValueForgotState(contextFactor, *res, &statelen);
float lmScore = ps ? m_implementation->GetValueGivenState(contextFactor, *res) : m_implementation->GetValueForgotState(contextFactor, *res);

// main loop
size_t endPos = std::min(startPos + GetNGramOrder() - 2
Expand All @@ -238,7 +237,7 @@ FFState* LanguageModel::Evaluate(
// add last factor
contextFactor.back() = &hypo.GetWord(currPos);

lmScore += m_implementation->GetValueGivenState(contextFactor, *res, &statelen);
lmScore += m_implementation->GetValueGivenState(contextFactor, *res);
}

// end of sentence
Expand Down
5 changes: 1 addition & 4 deletions moses/src/LanguageModelIRST.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,10 +184,8 @@ int LanguageModelIRST::GetLmID( const Factor *factor ) const
return ( factorId >= m_lmIdLookup.size()) ? m_unknownId : m_lmIdLookup[factorId];
}

float LanguageModelIRST::GetValue(const vector<const Word*> &contextFactor, State* finalState, unsigned int* len) const
float LanguageModelIRST::GetValue(const vector<const Word*> &contextFactor, State* finalState) const
{
unsigned int dummy;
if (!len) { len = &dummy; }
FactorType factorType = GetFactorType();

// set up context
Expand All @@ -212,7 +210,6 @@ float LanguageModelIRST::GetValue(const vector<const Word*> &contextFactor, Stat
prob = m_lmtb->clprob(codes,idx,NULL,NULL,&msp,&ilen);

if (finalState) *finalState=(State *) msp;
if (len) *len = ilen;

return TransformLMScore(prob);
}
Expand Down
2 changes: 1 addition & 1 deletion moses/src/LanguageModelIRST.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ class LanguageModelIRST : public LanguageModelPointerState
, FactorType factorType
, size_t nGramOrder);

virtual float GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL, unsigned int* len=0) const;
virtual float GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL) const;

void CleanUpAfterSentenceProcessing();
void InitializeBeforeSentenceProcessing();
Expand Down
7 changes: 3 additions & 4 deletions moses/src/LanguageModelImplementation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,17 +40,16 @@ namespace Moses
{
float LanguageModelImplementation::GetValueGivenState(
const std::vector<const Word*> &contextFactor,
FFState &state,
unsigned int* len) const
FFState &state) const
{
return GetValueForgotState(contextFactor, state, len);
return GetValueForgotState(contextFactor, state);
}

void LanguageModelImplementation::GetState(
const std::vector<const Word*> &contextFactor,
FFState &state) const
{
GetValueForgotState(contextFactor, state, NULL);
GetValueForgotState(contextFactor, state);
}

}
12 changes: 3 additions & 9 deletions moses/src/LanguageModelImplementation.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,22 +74,16 @@ class LanguageModelImplementation
* Specific implementation can return State and len data to be used in hypothesis pruning
* \param contextFactor n-gram to be scored
* \param state LM state. Input and output. state must be initialized. If state isn't initialized, you want GetValueWithoutState.
* \param len If non-null, the n-gram length is written here.
*/
virtual float GetValueGivenState(const std::vector<const Word*> &contextFactor, FFState &state, unsigned int* len = 0) const;
virtual float GetValueGivenState(const std::vector<const Word*> &contextFactor, FFState &state) const;

// Like GetValueGivenState but state may not be initialized (however it is non-NULL).
// For example, state just came from NewState(NULL).
virtual float GetValueForgotState(
const std::vector<const Word*> &contextFactor,
FFState &outState,
unsigned int* len = 0) const = 0;
virtual float GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState) const = 0;

//! get State for a particular n-gram. We don't care what the score is.
// This is here so models can implement a shortcut to GetValueAndState.
virtual void GetState(
const std::vector<const Word*> &contextFactor,
FFState &outState) const;
virtual void GetState(const std::vector<const Word*> &contextFactor, FFState &outState) const;

virtual FFState *GetNullContextState() const = 0;
virtual FFState *GetBeginSentenceState() const = 0;
Expand Down
3 changes: 1 addition & 2 deletions moses/src/LanguageModelInternal.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,7 @@ bool LanguageModelInternal::Load(const std::string &filePath
}

float LanguageModelInternal::GetValue(const std::vector<const Word*> &contextFactor
, State* finalState
, unsigned int* /*len*/) const
, State* finalState) const
{
const size_t ngram = contextFactor.size();
switch (ngram)
Expand Down
3 changes: 1 addition & 2 deletions moses/src/LanguageModelInternal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@ class LanguageModelInternal : public LanguageModelPointerState
, FactorType factorType
, size_t nGramOrder);
float GetValue(const std::vector<const Word*> &contextFactor
, State* finalState = 0
, unsigned int* len = 0) const;
, State* finalState = 0) const;
};

}
Expand Down
4 changes: 2 additions & 2 deletions moses/src/LanguageModelJoint.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ class LanguageModelJoint : public LanguageModelMultiFactor
return m_lmImpl->Load(filePath, m_implFactor, nGramOrder);
}

float GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState, unsigned int* len = NULL) const
float GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState) const
{
if (contextFactor.size() == 0)
{
Expand Down Expand Up @@ -117,7 +117,7 @@ class LanguageModelJoint : public LanguageModelMultiFactor
}

// calc score on chunked phrase
float ret = m_lmImpl->GetValueForgotState(jointContext, outState, len);
float ret = m_lmImpl->GetValueForgotState(jointContext, outState);

RemoveAllInColl(jointContext);

Expand Down
16 changes: 4 additions & 12 deletions moses/src/LanguageModelKen.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,8 @@ template <class Model> class LanguageModelKen : public LanguageModelSingleFactor
, FactorType factorType
, size_t nGramOrder);

float GetValueGivenState(const std::vector<const Word*> &contextFactor, FFState &state, unsigned int* len = 0) const;
float GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState, unsigned int* len=0) const;
float GetValueGivenState(const std::vector<const Word*> &contextFactor, FFState &state) const;
float GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState) const;
void GetState(const std::vector<const Word*> &contextFactor, FFState &outState) const;

FFState *GetNullContextState() const;
Expand Down Expand Up @@ -169,7 +169,7 @@ template <class Model> bool LanguageModelKen<Model>::Load(const std::string &fil
return true;
}

template <class Model> float LanguageModelKen<Model>::GetValueGivenState(const std::vector<const Word*> &contextFactor, FFState &state, unsigned int* len) const
template <class Model> float LanguageModelKen<Model>::GetValueGivenState(const std::vector<const Word*> &contextFactor, FFState &state) const
{
if (contextFactor.empty())
{
Expand All @@ -181,14 +181,10 @@ template <class Model> float LanguageModelKen<Model>::GetValueGivenState(const s
lm::ngram::State copied(realState);
lm::FullScoreReturn ret(m_ngram->FullScore(copied, new_word, realState));

if (len)
{
*len = ret.ngram_length;
}
return TransformLMScore(ret.prob);
}

template <class Model> float LanguageModelKen<Model>::GetValueForgotState(const vector<const Word*> &contextFactor, FFState &outState, unsigned int* len) const
template <class Model> float LanguageModelKen<Model>::GetValueForgotState(const vector<const Word*> &contextFactor, FFState &outState) const
{
if (contextFactor.empty())
{
Expand All @@ -200,10 +196,6 @@ template <class Model> float LanguageModelKen<Model>::GetValueForgotState(const
TranslateIDs(contextFactor, indices);

lm::FullScoreReturn ret(m_ngram->FullScoreForgotState(indices + 1, indices + contextFactor.size(), indices[0], static_cast<KenLMState&>(outState).state));
if (len)
{
*len = ret.ngram_length;
}
return TransformLMScore(ret.prob);
}

Expand Down
2 changes: 1 addition & 1 deletion moses/src/LanguageModelParallelBackoff.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ void LanguageModelParallelBackoff::CreateFactors()

}

float LanguageModelParallelBackoff::GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState, unsigned int* len) const
float LanguageModelParallelBackoff::GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState) const
{

static WidMatrix widMatrix;
Expand Down
2 changes: 1 addition & 1 deletion moses/src/LanguageModelParallelBackoff.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ VocabIndex GetLmID( const Factor *factor, FactorType ft ) const;

void CreateFactors();

float GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState, unsigned int* len = 0) const;
float GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState) const;
FFState *GetNullContextState() const;
FFState *GetBeginSentenceState() const;
FFState *NewState(const FFState *from) const;
Expand Down
7 changes: 2 additions & 5 deletions moses/src/LanguageModelRandLM.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,7 @@ randlm::WordID LanguageModelRandLM::GetLmID( const std::string &str ) const {
}

float LanguageModelRandLM::GetValue(const vector<const Word*> &contextFactor,
State* finalState, unsigned int* len) const {
unsigned int dummy; // is this needed ?
if (!len) { len = &dummy; }
State* finalState) const {
FactorType factorType = GetFactorType();
// set up context
randlm::WordID ngram[MAX_NGRAM_SIZE];
Expand All @@ -101,9 +99,8 @@ float LanguageModelRandLM::GetValue(const vector<const Word*> &contextFactor,
}
int found = 0;
float logprob = FloorScore(TransformLMScore(m_lm->getProb(&ngram[0], count, &found, finalState)));
*len = 0; // not available
//if (finalState)
// std::cerr << " = " << logprob << "(" << *finalState << ", " << *len <<")"<< std::endl;
// std::cerr << " = " << logprob << "(" << *finalState << ", " <<")"<< std::endl;
//else
// std::cerr << " = " << logprob << std::endl;
return logprob;
Expand Down
2 changes: 1 addition & 1 deletion moses/src/LanguageModelRandLM.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ class LanguageModelRandLM : public LanguageModelPointerState {
LanguageModelRandLM()
: m_lm(0) {}
bool Load(const std::string &filePath, FactorType factorType, size_t nGramOrder);
virtual float GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL, unsigned int* len=0) const;
virtual float GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL) const;
~LanguageModelRandLM() {
delete m_lm;
}
Expand Down
4 changes: 1 addition & 3 deletions moses/src/LanguageModelRemote.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ bool LanguageModelRemote::start(const std::string& host, int port) {
return true;
}

float LanguageModelRemote::GetValue(const std::vector<const Word*> &contextFactor, State* finalState, unsigned int* len) const {
float LanguageModelRemote::GetValue(const std::vector<const Word*> &contextFactor, State* finalState) const {
size_t count = contextFactor.size();
if (count == 0) {
if (finalState) *finalState = NULL;
Expand All @@ -76,7 +76,6 @@ float LanguageModelRemote::GetValue(const std::vector<const Word*> &contextFacto
cur = &cur->tree[event_word ? event_word : EOS];
if (cur->prob) {
if (finalState) *finalState = cur->boState;
if (len) *len = m_nGramOrder;
return cur->prob;
}
cur->boState = *reinterpret_cast<const State*>(&m_curId);
Expand Down Expand Up @@ -119,7 +118,6 @@ float LanguageModelRemote::GetValue(const std::vector<const Word*> &contextFacto
cur->prob = FloorScore(TransformLMScore(*reinterpret_cast<float*>(res)));
if (finalState) {
*finalState = cur->boState;
if (len) *len = m_nGramOrder;
}
return cur->prob;
}
Expand Down
2 changes: 1 addition & 1 deletion moses/src/LanguageModelRemote.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ class LanguageModelRemote : public LanguageModelPointerState {
public:
~LanguageModelRemote();
void ClearSentenceCache() { m_cache.tree.clear(); m_curId = 1000; }
virtual float GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0, unsigned int* len = 0) const;
virtual float GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0) const;
bool Load(const std::string &filePath
, FactorType factorType
, size_t nGramOrder);
Expand Down
6 changes: 2 additions & 4 deletions moses/src/LanguageModelSRI.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ float LanguageModelSRI::GetValue(VocabIndex wordId, VocabIndex *context) const
return FloorScore(TransformLMScore(p)); // log10->log
}

float LanguageModelSRI::GetValue(const vector<const Word*> &contextFactor, State* finalState, unsigned int *len) const
float LanguageModelSRI::GetValue(const vector<const Word*> &contextFactor, State* finalState) const
{
FactorType factorType = GetFactorType();
size_t count = contextFactor.size();
Expand All @@ -155,9 +155,7 @@ float LanguageModelSRI::GetValue(const vector<const Word*> &contextFactor, State
if (finalState) {
ngram[0] = lmId;
unsigned int dummy;
if (!len) { len = &dummy; }
*finalState = m_srilmModel->contextID(ngram, *len);
(*len)++;
*finalState = m_srilmModel->contextID(ngram, dummy);
}
return ret;
}
Expand Down
2 changes: 1 addition & 1 deletion moses/src/LanguageModelSRI.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ class LanguageModelSRI : public LanguageModelPointerState
, FactorType factorType
, size_t nGramOrder);

virtual float GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0, unsigned int* len = 0) const;
virtual float GetValue(const std::vector<const Word*> &contextFactor, State* finalState = 0) const;
};


Expand Down
4 changes: 2 additions & 2 deletions moses/src/LanguageModelSingleFactor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,9 @@ FFState *LanguageModelPointerState::NewState(const FFState *from) const
return new PointerState(from ? static_cast<const PointerState*>(from)->lmstate : NULL);
}

float LanguageModelPointerState::GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState, unsigned int* len) const
float LanguageModelPointerState::GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState) const
{
return GetValue(contextFactor, &static_cast<PointerState&>(outState).lmstate, len);
return GetValue(contextFactor, &static_cast<PointerState&>(outState).lmstate);
}

}
Expand Down
4 changes: 2 additions & 2 deletions moses/src/LanguageModelSingleFactor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,9 @@ class LanguageModelPointerState : public LanguageModelSingleFactor
virtual FFState *GetBeginSentenceState() const;
virtual FFState *NewState(const FFState *from = NULL) const;

virtual float GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState, unsigned int* len = 0) const;
virtual float GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState) const;

virtual float GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL, unsigned int* len=0) const = 0;
virtual float GetValue(const std::vector<const Word*> &contextFactor, State* finalState = NULL) const = 0;
};


Expand Down
4 changes: 2 additions & 2 deletions moses/src/LanguageModelSkip.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ class LanguageModelSkip : public LanguageModelSingleFactor
return m_lmImpl->NewState(from);
}

float GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState, unsigned int* len = 0) const
float GetValueForgotState(const std::vector<const Word*> &contextFactor, FFState &outState) const
{
if (contextFactor.size() == 0)
{
Expand Down Expand Up @@ -126,7 +126,7 @@ class LanguageModelSkip : public LanguageModelSingleFactor
std::reverse(chunkContext.begin(), chunkContext.end());

// calc score on chunked phrase
float ret = m_lmImpl->GetValueForgotState(chunkContext, outState, len);
float ret = m_lmImpl->GetValueForgotState(chunkContext, outState);

RemoveAllInColl(chunkContext);

Expand Down

0 comments on commit b3fa5e9

Please sign in to comment.