Lucene++ - a full-featured, c++ search engine
API Documentation


 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
StandardTokenizerImpl.h
Go to the documentation of this file.
1 // Copyright (c) 2009-2011 Alan Wright. All rights reserved.
3 // Distributable under the terms of either the Apache License (Version 2.0)
4 // or the GNU Lesser General Public License.
6 
7 #ifndef STANDARDTOKENIZERIMPL_H
8 #define STANDARDTOKENIZERIMPL_H
9 
10 #include "LuceneObject.h"
11 
12 namespace Lucene
13 {
15  {
16  public:
20 
21  virtual ~StandardTokenizerImpl();
22 
24 
25  protected:
27  static const int32_t ZZ_BUFFERSIZE;
28 
30  static const wchar_t ZZ_CMAP_PACKED[];
31  static const int32_t ZZ_CMAP_LENGTH;
32  static const int32_t ZZ_CMAP_PACKED_LENGTH;
33 
35  static const wchar_t* ZZ_CMAP();
36 
38  static const wchar_t ZZ_ACTION_PACKED_0[];
39  static const int32_t ZZ_ACTION_LENGTH;
40  static const int32_t ZZ_ACTION_PACKED_LENGTH;
41 
43  static const int32_t* ZZ_ACTION();
44 
46  static const wchar_t ZZ_ROWMAP_PACKED_0[];
47  static const int32_t ZZ_ROWMAP_LENGTH;
48  static const int32_t ZZ_ROWMAP_PACKED_LENGTH;
49 
51  static const int32_t* ZZ_ROWMAP();
52 
54  static const wchar_t ZZ_TRANS_PACKED_0[];
55  static const int32_t ZZ_TRANS_LENGTH;
56  static const int32_t ZZ_TRANS_PACKED_LENGTH;
57 
59  static const int32_t* ZZ_TRANS();
60 
61  // error codes
62  static const int32_t ZZ_UNKNOWN_ERROR;
63  static const int32_t ZZ_NO_MATCH;
64  static const int32_t ZZ_PUSHBACK_2BIG;
65 
66  static const wchar_t* ZZ_ERROR_MSG[];
67 
69  static const wchar_t ZZ_ATTRIBUTE_PACKED_0[];
70  static const int32_t ZZ_ATTRIBUTE_LENGTH;
71  static const int32_t ZZ_ATTRIBUTE_PACKED_LENGTH;
72 
74  static const int32_t* ZZ_ATTRIBUTE();
75 
78 
80  int32_t zzState;
81 
83  int32_t zzLexicalState;
84 
86  CharArray zzBuffer;
87 
89  int32_t zzMarkedPos;
90 
92  int32_t zzPushbackPos;
93 
95  int32_t zzCurrentPos;
96 
98  int32_t zzStartRead;
99 
101  int32_t zzEndRead;
102 
104  int32_t yyline;
105 
107  int32_t _yychar;
108 
110  int32_t yycolumn;
111 
113  bool zzAtBOL;
114 
116  bool zzAtEOF;
117 
118  public:
120  static const int32_t YYEOF;
121 
123  static const int32_t YYINITIAL;
124 
125  public:
126  int32_t yychar();
127 
129  void reset(ReaderPtr r);
130 
132  void getText(TokenPtr t);
133 
135  void getText(TermAttributePtr t);
136 
138  void yyclose();
139 
146  void yyreset(ReaderPtr reader);
147 
149  int32_t yystate();
150 
153  void yybegin(int32_t newState);
154 
156  String yytext();
157 
163  wchar_t yycharat(int32_t pos);
164 
166  int32_t yylength();
167 
172  void yypushback(int32_t number);
173 
176  int32_t getNextToken();
177 
178  protected:
180  bool zzRefill();
181 
191  void zzScanError(int32_t errorCode);
192  };
193 }
194 
195 #endif

clucene.sourceforge.net