Tanl Linguistic Pipeline

search/ixe/Common/PostingList.h

/*
**      IXE
**      Common/PostingList.h
** ----------------------------------------------------------------------
**      Copyright (c) 2000  Ideare SpA. All rights reserved.
**      Copyright (c) 2000  Giuseppe Attardi (attardi@di.unipi.it).
** ----------------------------------------------------------------------
**
**  This file is part of IXE.
**
**  IXE is free software; you can redistribute it and/or modify it
**  under the terms of the GNU General Public License, version 3,
**  as published by the Free Software Foundation.
**
**  IXE is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
**  GNU General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this program.  If not, see <http://www.gnu.org/licenses/>.
**  ----------------------------------------------------------------------
*/

#ifndef IXE_PostingList_H
#define IXE_PostingList_H

// Settings
#include "platform.h"

// standard
#include <cassert>
#include <cerrno>
#include <cstddef>                              /* for ptrdiff_t */
#include <fstream>
#include <iterator>
#include <sys/types.h>                          /* for off_t */

// local
#include "Common/eptacode.h"
#include "Common/lexicon.h"
#include "Common/Remap.h"
#include "include/Set.h"
#include "Index/Posting.h"

using namespace IXE::io;

namespace IXE {

#ifndef noSKIP_LIST
struct PostingOffset {
  unsigned int  offset;
  DocID         id;
};
#endif

class PostingList
{
public:

  typedef Size                  size_type;
  typedef ptrdiff_t             difference_type;
  typedef Posting               value_type;
  typedef value_type*           pointer;
  typedef value_type const*     const_pointer;
  typedef value_type&           reference;
  typedef value_type const&     const_reference;


  PostingList() : ptr_(0), size_(0) { }

  PostingList(byte const* ptr) : ptr_(ptr)
  {
    size_ = parseEptacode(ptr_);        // read term freq
  }

  PostingList(FileHandle* fh, off64_t offset, size_type size) {
    open(fh, offset, size);
  }

  //  PostingList(Lexicon::Cursor& cur) { open(cur); }

  PostingList(FileHandle* fh, Lexicon::const_iterator& cit) { open(fh, cit); }


  void          open(FileHandle* fh, off64_t offset, size_type size)
  {
    if (!postings.open(fh, offset, size))
      throw MmapError(string("Opening PostingList: ") + ::strerror(errno));
    ptr_ = (byte const*)postings.begin();
    size_ = parseEptacode(ptr_);        // read term freq
  }
  /*
  void          open(Lexicon::Cursor& cur)
  {
    off64_t postingStart = (*cur)->postingsOffset();
    open(cur.lexicon->postingsFile, postingStart,
         cur[1]->postingsOffset() - postingStart);
  }
  */
  void          open(FileHandle* fh, Lexicon::const_iterator& cit)
  {
    off64_t postingStart = (*cit)->postingsOffset();
    open(fh, postingStart, (Size)((*(cit + 1))->postingsOffset() - postingStart));
  }

  //  PostingList(PostingList const&); // forbid copy
  //  PostingList&      operator =(PostingList const&); // forbid assignment

  size_type     size() const { return size_; }


  class HitsCursor {
  public:

    HitsCursor() { }

    HitsCursor(short count, byte const* hitsOffset) :
      current(hitsOffset),
      count(count),
      pos(0) { }

    int                 next(unsigned min) {
      if (pos >= min)
        return pos;
      while (count) {
        count--;        // decrement only if positive
        pos += parseEptacode(current);
        if (pos >= min)
          return pos;
      }
      return noPosition;
    }

    inline byte const*  Current() { return current; }

    unsigned            nth(int n) {
      if (n > count)
        return noPosition;
      byte const* scan = current;
      unsigned res;
      while (n--)
        res = parseEptacode(scan);
      return res;
    }

    unsigned            operator *() { return pos; }

  private:
    byte const*         current;
    short               count;
    unsigned int        pos;    // not HitPosition, which might be short
  };


  class const_iterator;
  friend class const_iterator;

  class const_iterator : public 
  std::iterator<std::forward_iterator_tag, value_type, difference_type>
  {
    friend class PostingList;

  public:
    // used for end()
    const_iterator() : rest_(0) { }

    reference           operator *() { return posting; }
    pointer             operator ->() { return &posting; }

    const_iterator&     operator ++();
    // postfix version:
    const_iterator      operator ++(int) {
      const_iterator tmp = *this;
      ++*this;
      return tmp;
    }

    const_iterator&     next(DocID min);

    bool                atEnd() { return rest_ == 0; }

    size_type           size() const { return rest_; }

    friend inline bool
    operator ==(const_iterator const &i, const_iterator const &j) {
      return i.rest_ == j.rest_;
    }

    friend inline bool
    operator !=(const_iterator const &i, const_iterator const &j) {
      return !(i == j);
    }

    void                copyHits(std::fstream& o) {
      o << eptacode(hitlen - posting.termFrequency); // the delta
      // use hitsCursor, since c_ might have moved back to an earlier posting
      // because of remapping in remap_iterator.
      o.write((char*)hitsCursor.Current(), hitlen);
    }

    size_type           index() const { return size_ - rest_; }

    HitsCursor          hitsCursor;

  protected:
    const_iterator(size_type s, byte const *p) :
      rest_(s), c_(p)
#     ifndef noSKIP_LIST
      , size_(s), tablesz_(0)
#     endif
    {
      assert(p);
#     ifndef noSKIP_LIST        
      if (size_ > Min_Postings_Table) {
#       ifndef BYTE_ALIGN
        c_ = (byte*)ALIGN(((off_t)c_), sizeof(int));
#       endif
        table_ = (PostingOffset*)c_;
        // FIXME. Should be:
        //tablesz = (size_ + Postings_Segment_Size - 1)
        //  / Postings_Segment_Size - 1; // -1 since segment 0 is not present
        tablesz_ = size_ / Postings_Segment_Size;
        c_ += tablesz_ * sizeof(PostingOffset);
      } else
        table_ = 0;
#     endif // noSKIP_LIST

      rest_++;                  // decremented by ++()
      operator ++();
    }


    size_type           rest_;
    byte const*         c_;     

#   ifndef noSKIP_LIST

    size_type           tablesz_;
    size_type           size_;
    PostingOffset*      table_;
#   endif
    value_type          posting;
    Size                hitlen;
  };

  const_iterator        begin() const   { return const_iterator(size_, ptr_); }
  const_iterator        end() const     { return const_iterator(); }


  class remap_iterator : public const_iterator
  {
  public:
    remap_iterator(size_type s, byte const *p, int displace, Remap& remap) :
      // const_iterator(s, p), // replicate code, since we must call our ++()
      displace(displace),
#ifndef oldMerge
      remap(&remap),
      remapEnd(remap.end())
#else //oldMerge
      remapIt(remap.begin()),
      remapEnd(remap.end())
#endif //oldMerge
    {
      assert(p);
      rest_ = s;
      c_ = p;
#     ifndef noSKIP_LIST
      size_ = s;
      tablesz_ = 0;
#     endif
#     ifndef noSKIP_LIST        
      if (size_ > Min_Postings_Table) {
#       ifndef BYTE_ALIGN
        c_ = (byte*)ALIGN(((off_t)c_), sizeof(int));
#       endif
        table_ = (PostingOffset*)c_;
        // FIXME. Should be:
        //tablesz = (size_ + Postings_Segment_Size - 1)
        //  / Postings_Segment_Size - 1; // -1 since segment 0 is not present
        tablesz_ = size_ / Postings_Segment_Size;
        c_ += tablesz_ * sizeof(PostingOffset);
      } else
        table_ = 0;
#     endif // noSKIP_LIST

      // List: (3, 6, 7, 8)
      // Remap: (2 -> 7), (4 -> 9), (5 -> 6)
      // rest_: 4, c_: 3, 6, 7, 8       posting: -
      ++rest_;
      operator ++();
      // rest_: 4, c_: 3, 6, 7, 8       posting: 7 (remapped to 2)
    }

    remap_iterator&     operator ++();

  private:
    int                         displace;
#ifndef oldMerge
    Remap * remap;
    Remap::const_iterator       remapEnd;
#else //oldMerge
    Remap::const_iterator       remapIt;
    Remap::const_iterator       remapEnd;
    Set<DocID>                  remapped;
#endif //oldMerge
  };

  remap_iterator        remap_begin(int displace, Remap& remap) const {
    return remap_iterator(size_, ptr_, displace, remap); }

private:
  MappedFileView        postings;
  byte const*           ptr_;
  mutable size_type     size_;
};

} // namespace IXE

#endif  /* IXE_PostingList_H */

 

Copyright © 2005-2011 G. Attardi. Generated on 4 Mar 2011 by doxygen 1.6.1.