View Javadoc

1   /*
2    * Token.java - Generic token
3    * Copyright (C) 1998, 1999 Slava Pestov
4    *
5    * You may use and modify this package for any purpose. Redistribution is
6    * permitted, in both source and binary form, provided that this notice
7    * remains intact in all source distributions of this package.
8    */
9   
10  package org.syntax.jedit.tokenmarker;
11  
12  /***
13   * A linked list of tokens. Each token has three fields - a token
14   * identifier, which is a byte value that can be looked up in the
15   * array returned by <code>SyntaxDocument.getColors()</code>
16   * to get a color value, a length value which is the length of the
17   * token in the text, and a pointer to the next token in the list.
18   *
19   * @author Slava Pestov
20   * @version $Id$
21   */
22  
23  public class Token
24  {
25  	/***
26  	 * Normal text token id. This should be used to mark
27  	 * normal text.
28  	 */
29  	public static final byte NULL = 0;
30  
31  	/***
32  	 * Comment 1 token id. This can be used to mark a comment.
33  	 */
34  	public static final byte COMMENT1 = 1;
35  
36  	/***
37  	 * Comment 2 token id. This can be used to mark a comment.
38  	 */
39  	public static final byte COMMENT2 = 2;
40  
41  	
42  	/***
43  	 * Literal 1 token id. This can be used to mark a string
44  	 * literal (eg, C mode uses this to mark "..." literals)
45  	 */
46  	public static final byte LITERAL1 = 3;
47  
48  	/***
49  	 * Literal 2 token id. This can be used to mark an object
50  	 * literal (eg, Java mode uses this to mark true, false, etc)
51  	 */
52  	public static final byte LITERAL2 = 4;
53  
54  	/***
55  	 * Label token id. This can be used to mark labels
56  	 * (eg, C mode uses this to mark ...: sequences)
57  	 */
58  	public static final byte LABEL = 5;
59  
60  	/***
61  	 * Keyword 1 token id. This can be used to mark a
62  	 * keyword. This should be used for general language
63  	 * constructs.
64  	 */
65  	public static final byte KEYWORD1 = 6;
66  
67  	/***
68  	 * Keyword 2 token id. This can be used to mark a
69  	 * keyword. This should be used for preprocessor
70  	 * commands, or variables.
71  	 */
72  	public static final byte KEYWORD2 = 7;
73  
74  	/***
75  	 * Keyword 3 token id. This can be used to mark a
76  	 * keyword. This should be used for data types.
77  	 */
78  	public static final byte KEYWORD3 = 8;
79  
80  	/***
81  	 * Operator token id. This can be used to mark an
82  	 * operator. (eg, SQL mode marks +, -, etc with this
83  	 * token type)
84  	 */
85  	public static final byte OPERATOR = 9;
86  
87  	/***
88  	 * Invalid token id. This can be used to mark invalid
89  	 * or incomplete tokens, so the user can easily spot
90  	 * syntax errors.
91  	 */
92  	public static final byte INVALID = 10;
93  
94  	/***
95  	 * The total number of defined token ids.
96  	 */
97  	public static final byte ID_COUNT = 11;
98  
99  	/***
100 	 * The first id that can be used for internal state
101 	 * in a token marker.
102 	 */
103 	public static final byte INTERNAL_FIRST = 100;
104 
105 	/***
106 	 * The last id that can be used for internal state
107 	 * in a token marker.
108 	 */
109 	public static final byte INTERNAL_LAST = 126;
110 
111 	/***
112 	 * The token type, that along with a length of 0
113 	 * marks the end of the token list.
114 	 */
115 	public static final byte END = 127;
116 
117 	/***
118 	 * The length of this token.
119 	 */
120 	public int length;
121 
122 	/***
123 	 * The id of this token.
124 	 */
125 	public byte id;
126 
127 	/***
128 	 * The next token in the linked list.
129 	 */
130 	public Token next;
131 
132 	/***
133 	 * Creates a new token.
134 	 * @param length The length of the token
135 	 * @param id The id of the token
136 	 */
137 	public Token(int length, byte id)
138 	{
139 		this.length = length;
140 		this.id = id;
141 	}
142 
143 	/***
144 	 * Returns a string representation of this token.
145 	 */
146 	public String toString()
147 	{
148 		return "[id=" + id + ",length=" + length + "]";
149 	}
150 }