Lucene++ - a full-featured, c++ search engine
API Documentation


 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
Tokenizer.h
Go to the documentation of this file.
1 // Copyright (c) 2009-2011 Alan Wright. All rights reserved.
3 // Distributable under the terms of either the Apache License (Version 2.0)
4 // or the GNU Lesser General Public License.
6 
7 #ifndef TOKENIZER_H
8 #define TOKENIZER_H
9 
10 #include "TokenStream.h"
11 
12 namespace Lucene
13 {
20  class LPPAPI Tokenizer : public TokenStream
21  {
22  protected:
24  Tokenizer();
25 
27  Tokenizer(ReaderPtr input);
28 
31 
33  Tokenizer(AttributeFactoryPtr factory, ReaderPtr input);
34 
37 
39  Tokenizer(AttributeSourcePtr source, ReaderPtr input);
40 
41  public:
42  virtual ~Tokenizer();
43 
45 
46  protected:
48  ReaderPtr input;
50 
51  public:
53  virtual void close();
54 
60  virtual int32_t correctOffset(int32_t currentOff);
61 
62  using TokenStream::reset;
63 
66  virtual void reset(ReaderPtr input);
67  };
68 }
69 
70 #endif

clucene.sourceforge.net