Tanl Linguistic Pipeline

search/ixe/Common/lexicon.h

/*
**      IXE
**      Common/lexicon.h
** ----------------------------------------------------------------------
**      Copyright (c) 2000  Ideare SpA. All rights reserved.
**      Copyright (c) 2000  Giuseppe Attardi (attardi@di.unipi.it).
** ----------------------------------------------------------------------
**
**  This file is part of IXE.
**
**  IXE is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  IXE is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/


#ifndef IXE_lexicon_H
#define IXE_lexicon_H

// Settings
#include "platform.h"

// standard
#include <algorithm>            /* for binary_search(), etc */
#include <iterator>
#include <string>

// local
#include "include/config.h"
#include "include/ixe.h"
#include "io/FileHandle.h"
#include "Common/IndexTable.h"
#include "Common/LexEntry.h"

using namespace IXE::io;

namespace IXE {

class TermInfo {
public:
  TermID        ID;
  Count         freq;           // document frequency
};

class PostingList;

class Lexicon : public IndexTable<LexEntry>
{
 public:

  typedef std::pair<const_iterator, const_iterator> Range;

  /*
  ~Lexicon();                   // close file
  bool          serialize(ostream &); // dump content to output stream

  termInfo&     get(const Term& word);
  bool          exists(const Term& word) const;
  */

  void          access(mappedFile& mf, off32_t& offset);

  void          accessPostings(FileHandle* fh) {
    postingsFile = fh;
  }

  Size  getWord(TermID termID, char* term, Size size) const;

  const_iterator find(char const* word);

  const_iterator find(byte* word, Size len);

  Range         findPrefix(char const * term) {
    return findPrefix(term, ::strlen(term));
  }

  Range         findPrefix(char const * term, Size len);

  Range         findPrefix(wchar_t const * term, Size len) {
    return findPrefix((char const*)term, len * sizeof(wchar_t));
  }

  Count         frequency(char const* word);

  friend class Cursor;

  class Cursor
  {
  public:
    Cursor() { }

    Cursor(Lexicon* l, int index = 0) :
      lexicon(l), word_index(index - 1), bigram_index(0) { }

    bool                atEnd() const {
      // assert(lexicon != 0);
      return word_index >= (int)lexicon->size();
    }

    bool                hasNext() const {
      // assert(lexicon != 0);
      // cast to int since word_index may be negative
      return word_index < (int)lexicon->size() - 1;
    }

    LexEntry const*     next() {
      return (*lexicon)[++word_index];
    }

    LexEntry const*     previous() {
      return (*lexicon)[--word_index];
    }

    char const*         next(char const* minWord) {
      if (::strcmp(word(), minWord) < 0 && word_index)
        word_index++;
      return word();
    }

    char const*         previous(char const* maxWord) {
      if (::strcmp(word(), maxWord) >= 0 && word_index)
        word_index--;
      return word();
    }

    long                get() { return word_index; }

    void                set(long index) {
      word_index = index;
      bigram_index = 0;
    }

    void                reset() { word_index = 0; }

    LexEntry const*     operator*() {
      return (*lexicon)[word_index];
    }

    LexEntry const*     operator [](int i) {
      return (*lexicon)[word_index + i];
    }

    char const*         word();

    PostingList         postingList();
    void                postingList(PostingList&);

  private:
    Lexicon*            lexicon;
    long                word_index; // starts at -1
    unsigned int        bigram_index; // not short! (see word())
    char                term[2*WordMaxSize+1]; // include null
  };

  Cursor                cursor(int i = 0) { return Cursor(this, i); }

  BigramTable           bigrams;

  FileHandle*           postingsFile;

private:
  friend struct EntryCompare;

  char const*   getPostfix(LexEntry const* c) const {
    return begin_ + c->word;
  }

# ifdef LEXLEN

  char const*   getPostfix(LexEntry const* c, Size& len) const {
    len = c->len;
    return begin_ + c->word;
  }

  char const*   getPostfix(TermID termID, Size& len) const {
    len = table[termID].len;
    return begin_ + table[termID].word;
  }

  char const*   getPostfix(const_iterator& cit, Size& len) const {
    len = table[cit.position()].len;
    return begin_ + table[cit.position()].word;
  }
# endif

  char const*   getPostfix(TermID termID) const {
    return begin_ + table[termID].word;
  }

  char const*   getPostfix(const_iterator& cit) const {
    return begin_ + table[cit.position()].word;
  }
};

} // namespace IXE

#endif  /* IXE_lexicon_H */

 

Copyright © 2005-2011 G. Attardi. Generated on 4 Mar 2011 by doxygen 1.6.1.