Object
# File lib/webrobots/robotstxt.rb, line 527 def initialize(site, records, options = nil) @timestamp = Time.now @site = site @options = options || {} @last_checked = nil @error = @options[:error] @target = @options[:target] @sitemaps = @options[:sitemaps] || [] if records && !records.empty? @records, defaults = [], [] records.each { |record| if record.default? defaults << record elsif !@target || record.match?(@target) @records << record end } @records.concat(defaults) else @records = [] end end
# File lib/webrobots/robotstxt.rb, line 578 def allow?(request_uri, user_agent = nil) record = find_record(user_agent) or return true allow = record.allow?(request_uri) if @last_checked and delay = record.delay delay -= Time.now - @last_checked sleep delay if delay > 0 end @last_checked = Time.now return allow end
Generated with the Darkfish Rdoc Generator 2.