class RMMSeg::Ferret::Tokenizer

The Tokenizer tokenize text with RMMSeg::Algorithm.

Public Class Methods

new(str) click to toggle source

Create a new Tokenizer to tokenize text

# File lib/rmmseg/ferret.rb, line 34
def initialize(str)
  self.text = str
end

Public Instance Methods

next() click to toggle source

Get next token

# File lib/rmmseg/ferret.rb, line 39
def next
  tok = @algor.next_token
  if tok.nil?
    return nil
  else
    @token.text = tok.text
    @token.start = tok.start
    @token.end = tok.end
    return @token
  end
end
text() click to toggle source

Get the text being tokenized

# File lib/rmmseg/ferret.rb, line 52
def text
  @text
end
text=(str) click to toggle source

Set the text to be tokenized

# File lib/rmmseg/ferret.rb, line 57
def text=(str)
  @token = ::Ferret::Analysis::Token.new("", 0, 0)
  @text = str
  @algor = Algorithm.new(@text)
end