Tanl Linguistic Pipeline

tag/Tanlpos/hmm_tagger.h

/*
**  Tanl
**  tag/TanlPos/hmm_tagger.h
**  ----------------------------------------------------------------------
**  Copyright (c) 2005  Giuseppe Attardi (attardi@di.unipi.it).
**  ----------------------------------------------------------------------
**
**  This file is part of Tanl.
**
**  Tanl is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  Tanl is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

#ifndef Tanl_POS_hmm_tagger_H
#define Tanl_POS_hmm_tagger_H

#include "linear_interpolated_lm.h"
#include "lexicon.h"
#include "MorphTable.h"
#include "Viterbi.h"
#include "Vocab.h"
#include "SuffixGuesser.h"

// standard

namespace Tanl { namespace POS {
    
typedef std::pair<int, double> TagWeight;
typedef std::vector<TagWeight> TagWeights;

struct Model : public HMM
{
  Model(int tag_order, int emission_order);
    
  Model(char const* model_file, char const* morph_file);

  std::vector<Transition> transitions(State& from, Observation& obs);

  void serialize(std::istream&);
  void serialize(std::ostream&);

  // persistent
  int tag_order;
  int emission_order;
  ProbLM<int> tag_lm;
  ProbLM<std::string> obs_lm;
  ProbLM<std::string> spec_lm;
  Lexicon obs_lex;
  Lexicon spec_lex;
  Vocab tag_vocab;
  SuffixGuesser low_suffixes;
  SuffixGuesser upp_suffixes;
  int eos;
  int bos;
  std::vector<double> apriori_tag_probs;
  double theta;

  // volatile
  int max_guessed_tags;
  std::vector<double> suffix_accu;
    
private:
  std::vector<Transition> tags2transition(std::vector<std::string>& tags, State& from, double emission);
    
  Morphtable            morphtable;
};

class HmmTagger
{
    
public:
  HmmTagger(int tag_order, int emission_order);
    
  HmmTagger(char const* model_file, char const* morph_file,
            int max_guessed_tags, double logtheta);
    
  bool add_sentence(std::vector<std::string>& words,
                    std::vector<std::string>& tags);
        
  void calculate_probs();
        
  void build_suffixtries(int maxfreq, int maxlength);
        
  void print_stat();
        
  void save(char const* file_name);
        
  void tag_sentence(std::vector<std::string>& words,
                    std::vector<std::string>& tags,
                    std::vector<HMM::Observation>* observations = 0);

private:
            
  struct train_stat {
    int tokens;
    int sentences;
    int types;
    int rare_low;
    int rare_upp;
                
    train_stat() :
      tokens(0),
      sentences(0),
      types(0),
      rare_low(0),
      rare_upp(0)
    { }
  };
            
  Model m;
  train_stat    stat;
  Viterbi       StateViterbi;
            
  // volatile
  int           max_guessed_tags;
  double        logtheta;
            
};
    
} // namespace POS
} // namespace Tanl

#endif //  Tanl_POS_hmm_tagger_H

 

Copyright © 2005-2011 G. Attardi. Generated on 4 Mar 2011 by doxygen 1.6.1.