View Javadoc

1   /***
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3   */
4   package net.sourceforge.pmd.cpd;
5   
6   import net.sourceforge.pmd.TargetJDK1_4;
7   import net.sourceforge.pmd.ast.JavaParserTokenManager;
8   import net.sourceforge.pmd.ast.Token;
9   
10  import java.io.StringReader;
11  
12  public class JavaTokenizer implements Tokenizer {
13  
14      public void tokenize(SourceCode tokens, Tokens tokenEntries) {
15          StringBuffer sb = tokens.getCodeBuffer();
16  
17          /*
18          I'm doing a sort of State pattern thing here where
19          this goes into "discarding" mode when it hits an import or package
20          keyword and goes back into "accumulate mode when it hits a semicolon.
21          This could probably be turned into some objects.
22          */
23          JavaParserTokenManager tokenMgr = new TargetJDK1_4().createJavaParserTokenManager(new StringReader(sb.toString()));
24          Token currToken = tokenMgr.getNextToken();
25          boolean discarding = false;
26          while (currToken.image != "") {
27              if (currToken.image.equals("import") || currToken.image.equals("package")) {
28                  discarding = true;
29                  currToken = tokenMgr.getNextToken();
30                  continue;
31              }
32  
33              if (discarding && currToken.image.equals(";")) {
34                  discarding = false;
35              }
36  
37              if (discarding) {
38                  currToken = tokenMgr.getNextToken();
39                  continue;
40              }
41  
42              if (!currToken.image.equals(";")) {
43                  tokenEntries.add(new TokenEntry(currToken.image, tokens.getFileName(), currToken.beginLine));
44              }
45  
46              currToken = tokenMgr.getNextToken();
47          }
48          tokenEntries.add(TokenEntry.getEOF());
49      }
50  }