Parent

WebRobots::RobotsTxt

Constants

DISALLOW_ALL

Attributes

error[RW]
site[R]
sitemaps[R]
timestamp[R]

Public Class Methods

new(site, records, options = nil) click to toggle source
# File lib/webrobots/robotstxt.rb, line 527
def initialize(site, records, options = nil)
  @timestamp = Time.now
  @site = site
  @options = options || {}
  @last_checked = nil

  @error = @options[:error]
  @target = @options[:target]
  @sitemaps = @options[:sitemaps] || []

  if records && !records.empty?
    @records, defaults = [], []
    records.each { |record|
      if record.default?
        defaults << record
      elsif !@target || record.match?(@target)
        @records << record
      end
    }
    @records.concat(defaults)
  else
    @records = []
  end
end
unfetchable(site, reason, target = nil) click to toggle source
# File lib/webrobots/robotstxt.rb, line 599
def self.unfetchable(site, reason, target = nil)
  Parser.new(target).parse(DISALLOW_ALL, site).tap { |robots_txt|
    robots_txt.error = reason
  }
end

Public Instance Methods

allow?(request_uri, user_agent = nil) click to toggle source
# File lib/webrobots/robotstxt.rb, line 578
def allow?(request_uri, user_agent = nil)
  record = find_record(user_agent) or return true
  allow = record.allow?(request_uri)
  if @last_checked and delay = record.delay
    delay -= Time.now - @last_checked
    sleep delay if delay > 0
  end
  @last_checked = Time.now
  return allow
end
error!() click to toggle source
# File lib/webrobots/robotstxt.rb, line 555
def error!
  raise @error if @error
end
options(user_agent = nil) click to toggle source
# File lib/webrobots/robotstxt.rb, line 589
def options(user_agent = nil)
  record = find_record(user_agent) or return {}
  record.options
end

[Validate]

Generated with the Darkfish Rdoc Generator 2.