Tanl Linguistic Pipeline

tag/SST/SstFeatureExtractor.h

/*
**  Tanl
**  SST/SstFeatureExtractor.h
**  ----------------------------------------------------------------------
**  Copyright (c) 2005  Giuseppe Attardi (attardi@di.unipi.it).
**  ----------------------------------------------------------------------
**
**  This file is part of Tanl.
**
**  Tanl is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  Tanl is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

#ifndef Tanl_SST_SstFeatureExtractor_H
#define Tanl_SST_SstFeatureExtractor_H

// Library
#include "text/WordIndex.h"
#include "text/WordSet.h"
#include "text/Suffixes.h"

// Local
#include "TokenCategorizer.h"
#include "Corpus/Token.h"
#include "classifier/Classifier.h"
#include "classifier/FeatureExtractor.h"

// Standard
#include <cstring>
#include <string>
#include <vector>

namespace Tanl {
namespace SST {

// Representation of feature model.
struct FeatureSpec {
  FeatureSpec(char const* name, char const* tokens);
  char const* name;             
  std::vector<int>      tokens; 
};

struct FeatureSpecs : public std::vector<FeatureSpec>
{
  FeatureSpecs();
};

//======================================================================

// Used to get the encoding of a tag corresponding to previous/next positions.
typedef unordered_map<char const*, std::string> TagSet;

#define NUM_CLASSES 46

struct Resources
{
  static char const* classNames[];
  
  Text::WordIndex       classId;

  static const int nClasses = NUM_CLASSES;

  Resources() {}

  Resources(std::string& locale) : language(locale.c_str())
  { }

  Resources(std::string& resourceDir, std::string& locale);

  char const*           language;

  // Token Types
  char const*   typeName(EntityType et);
  TagSet        prevTokenType; // feature Type for prev word
  TagSet        nextTokenType; // feature Type for next word

  Tanl::Text::NormWordSet       FWL;    // Frequent Word List

  Tanl::Text::NormWordSet       designators[NUM_CLASSES]; // func

  Tanl::Text::NormWordSet       preBigrams[NUM_CLASSES]; // ubi

  Tanl::Text::Suffixes          suffixes[NUM_CLASSES]; // wordSuf

  Tanl::Text::NormWordSet       lastWords[NUM_CLASSES]; // classSuf

  Tanl::Text::NormWordSet       lowerInterm[NUM_CLASSES];

  void load(std::string& resourceDir);

  template<class WordSet>
  void          load(WordSet* sets, char const* file);
};

//======================================================================
//======================================================================

class SstFeatureExtractor :
 public Classifier::FeatureExtractor<Classifier::Features, const int>
{
public:
  SstFeatureExtractor(Resources& resources);

  ~SstFeatureExtractor() { reset(); }

  void          analyze(Sentence* sent, int zone);

  void  extract(Classifier::Features& feats, const int& pos);

  void  reset();

  void  classified(int position, char const* className);

protected:
  Resources&            resources;
  bool                  insideQuotes;
  TokenCategorizer      tokenCategorizer;       
  std::vector<EntityType>       tokenTypes; 

  Sentence*             sentence; 

  // Global Maps

  unordered_map<string, bool>   capitalized; // firstInitCap

  Tanl::Text::NormWordSet       prevClass[NUM_CLASSES]; // NCPrevOccur

  Tanl::Text::NormWordSet       otherLast[NUM_CLASSES]; // otherNCS

  Tanl::Text::NormWordSet       acronyms;

private:
  void designated(char const* word, char const* ssTag);
  FeatureSpecs  featureSpecs;
  int           zone;   // zone within document
};

bool allUpper(char const* s);
bool mixedCase(char const* s);
bool noLetter(char const* s);
bool containsDigit(char const* s);
bool allDigits(char const* s);
bool allQuotes(char const* s);

extern char const* UnknownTag;
extern std::string const otherPrevLoc;
extern std::string const otherPrevOrg;
extern std::string const otherPrevPers;

} // namespace SST
} // namespace Tanl

#endif // Tanl_SST_SstFeatureExtractor_H

 

Copyright © 2005-2011 G. Attardi. Generated on 4 Mar 2011 by doxygen 1.6.1.