Lucene++ - a full-featured, c++ search engine
API Documentation
Main Page
Related Pages
Namespaces
Data Structures
Files
File List
Globals
All
Data Structures
Namespaces
Files
Functions
Variables
Typedefs
Enumerations
Enumerator
Macros
Pages
include
Tokenizer.h
Go to the documentation of this file.
1
// Copyright (c) 2009-2011 Alan Wright. All rights reserved.
3
// Distributable under the terms of either the Apache License (Version 2.0)
4
// or the GNU Lesser General Public License.
6
7
#ifndef TOKENIZER_H
8
#define TOKENIZER_H
9
10
#include "
TokenStream.h
"
11
12
namespace
Lucene
13
{
20
class
LPPAPI
Tokenizer
:
public
TokenStream
21
{
22
protected
:
24
Tokenizer
();
25
27
Tokenizer
(
ReaderPtr
input);
28
30
Tokenizer
(
AttributeFactoryPtr
factory);
31
33
Tokenizer
(
AttributeFactoryPtr
factory,
ReaderPtr
input);
34
36
Tokenizer
(
AttributeSourcePtr
source);
37
39
Tokenizer
(
AttributeSourcePtr
source,
ReaderPtr
input);
40
41
public
:
42
virtual
~
Tokenizer
();
43
44
LUCENE_CLASS
(
Tokenizer
);
45
46
protected
:
48
ReaderPtr
input;
49
CharStreamPtr
charStream
;
50
51
public
:
53
virtual
void
close();
54
60
virtual
int32_t correctOffset(int32_t currentOff);
61
62
using
TokenStream::reset
;
63
66
virtual
void
reset(
ReaderPtr
input);
67
};
68
}
69
70
#endif
clucene.sourceforge.net