Tanl Linguistic Pipeline

split/Tokenizer/PtbTokenizer.h

/*
**  Tanl
**  split/Tokenizer/PtbTokenizer.h
**  ----------------------------------------------------------------------
**  Copyright (c) 2008  Giuseppe Attardi (attardi@di.unipi.it).
**  ----------------------------------------------------------------------
**
**  This file is part of Tanl.
**
**  Tanl is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  Tanl is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

#ifndef Tanl_PtbTokenizer_H
#define Tanl_PtbTokenizer_H

#ifndef yyFlexLexer
// avoid redefinition, when included from PtbTokenizer.lex
# define yyFlexLexer ptbFlexLexer
# include "FlexLexer.h"
#endif

// standard
#include <stdlib.h>
#include <string.h>
#include <cstdio>
#include <fstream>

// local
#include "Enumerator.h"

using namespace std;

namespace Tanl {

class PtbScanner : public yyFlexLexer
{

 public:

  enum TokenType {
    Eof  =  0,
    Word = 200,
    Abbrev,
    Date,
    Email,
    None,
    Number,
    Phone,
    Special,
    Tag,
    Url,
    Punct,
    Eos
  };

  struct Token {
    Token(char const* text = "", TokenType type = Eof) :
      text(strdup(text)), type(type) { }
    
    Token(std::string const& text, TokenType type = Eof) :
      text(strdup(text.c_str())), type(type) { }
    
    char const* text;           
    TokenType type;             

    Token const& operator =(Token const& o) {
      if (this != &o) {
        delete text;
        text = strdup(o.text);
        type = o.type;
      }
      return *this;
    }

   ~Token() { free((void*)text); }

  };

  Token scan();                 // i.e. YY_DECL
};

std::ostream& operator <<(std::ostream& os, PtbScanner::Token const& tk) {
  return os << tk.text;
}

#undef YY_DECL
#define YY_DECL Tanl::PtbScanner::Token Tanl::PtbScanner::scan()
#define yylval token.text

class PtbTokenizer : public Tanl::Enumerator<PtbScanner::Token const*>
{
 public:

  PtbTokenizer(std::istream* is, char const* lang = 0) :
    stream(*is)
  {
    scanner.switch_streams(&stream, 0);
  }

  PtbScanner::Token const*      Current() { return &token; }

  bool          MoveNext() {
    token = scanner.scan();
    return token.type != PtbScanner::Eof;
  }

  void          Reset() {
    stream.seekg(0);
    scanner.switch_streams(&stream, 0);
  }

 private:
  PtbScanner    scanner;
  PtbScanner::Token token;              
  std::istream& stream;
};

} // namespace tanl

#endif // Tanl_PtbTokenizer_H

 

Copyright © 2005-2011 G. Attardi. Generated on 4 Mar 2011 by doxygen 1.6.1.