Tanl Linguistic Pipeline

parse/desr/src/Tokenizer.h

/*
**  DeSR
**  src/Tokenizer.h
**  ----------------------------------------------------------------------
**  Copyright (c) 2008  Giuseppe Attardi (attardi@di.unipi.it).
**  ----------------------------------------------------------------------
**
**  This file is part of DeSR.
**
**  DeSR is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  DeSR is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

#ifndef DeSR_Tokenizer_H
#define DeSR_Tokenizer_H

// settings

// standard
#include <stdlib.h>
#include <string.h>
#include <cstdio>
#include <fstream>

// local
#include "Enumerator.h"
#include "Serializable.h"

namespace Parser {

class Scanner : public ptbFlexLexer
{

 public:

  enum TokenType {
    Eof  =  0,
    Word = 200,
    Abbrev,
    Date,
    Number,
    Phone,
    Tag,
    Url,
    Punct
  };

  static char const* TypeName[];

  // can't inherit from Serializable, because SWIG can't handle it
  // nested class with virtual methods.
  struct Token { // : public Serializable {
    Token(char const* text = 0, TokenType type = (TokenType)0) :
      text(text), type(type) { }
    
    char const* text;           
    TokenType type;             

    Token const& operator =(Token const& o) {
      if (this != &o) {
        delete text;
        text = strdup(o.text);
        type = o.type;
      }
      return *this;
    }

   ~Token() { free((void*)text); }

    void        serialize(std::ostream& stream);

  };

  Token scan();                 // i.e. YY_DECL
};

#undef YY_DECL
#define YY_DECL Parser::Scanner::Token Parser::Scanner::scan()
#define yylval token.text

class Tokenizer : public Tanl::Enumerator<Scanner::Token const*>
{
 public:

  Tokenizer(std::istream* is, char const* lang = 0);

  Scanner::Token const* Current();

  bool          MoveNext();

  void          Reset();

 private:
  Scanner       scanner;
  Scanner::Token token;         
  std::istream& stream;
};

} // namespace Parser

#endif // DeSR_Tokenizer_H

 

Copyright © 2005-2011 G. Attardi. Generated on 4 Mar 2011 by doxygen 1.6.1.