httprb · pezra · Mar 18, 2014 · Mar 18, 2014 · Mar 18, 2014 · Mar 19, 2014
diff --git a/README.md b/README.md
@@ -249,6 +249,27 @@ There's a little more to it, but that's the core idea!
 * [Full parallel HTTP fetcher example](https://github.com/httprb/http.rb/wiki/Parallel-requests-with-Celluloid%3A%3AIO)
 * See also: [Celluloid::IO](https://github.com/celluloid/celluloid-io)
 
+### Caching
+
+http.rb provides caching of HTTP request (per
+[RFC 7234](https://tools.ietf.org/html/rfc7234)) when configured to do
+so.
+
+```ruby
+require 'http'
+
+http = HTTP.with_cache(metastore:   "file:/var/cache/my-app-http/meta",
+                       entitystore: "file:/var/cache/my-app-http/entity")
+
+http.get("http://example.com/")   # makes request
+http.get("http://example.com/")   # skips making request and returns
+                                  # previously cached response
+```
+
+http.rb's caching is backed by
+[rack-cache's excellent storage subsystem](http://rtomayko.github.io/rack-cache/storage.html). Any
+storage URL supported by rack-cache is supported by http.rb's cache.
+
 Supported Ruby Versions
 -----------------------
 

diff --git a/Rakefile b/Rakefile
@@ -23,7 +23,7 @@ end
 require "yardstick/rake/verify"
 Yardstick::Rake::Verify.new do |verify|
   verify.require_exact_threshold = false
-  verify.threshold = 58
+  verify.threshold = 58.1
 end
 
 task :default => [:spec, :rubocop, :verify_measurements]
diff --git a/expected_results b/expected_results
@@ -0,0 +1,55 @@
+
+POST /1416179329 HTTP/1.1
+Host: localhost:8080
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:32.0) Gecko/20100101 Firefox/32.0 Iceweasel/32.0
+Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
+Accept-Language: en-US,en;q=0.5
+Accept-Encoding: gzip, deflate
+DNT: 1
+Referer: http://localhost:8080/1416179301
+Cookie: __utma=111872281.776418867.1390272758.1415735077.1415742340.6; __utmz=111872281.1390272758.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); _jsuid=2367279670; sid=1bad10a0201c253cfaf0bfebe50405fef82b2267; _vidiq.sess=BAh7CUkiDHVzZXJfaWQGOgZFVCIdNTQ1YThlZjg5N2YwMTQwNzk1MDAwMDAxSSIPY3JlYXRlZF9hdAY7AFRsKwcjj1pUSSIPc2Vzc2lvbl9pZAY7AFRJIiU3YTE4NGQ0YTc1ZTI0YzEwOGFhNjZmYzNkOTQ3NjkzNAY7AFRJIhFsYXN0X3JlcXVlc3QGOwBUbCsHspNaVA%3D%3D--2d06b2c91f7eddd469d2781ab14a57759dc44543; locale=en-US; _vidiq.auth=1; utag_main=v_id:0149448f5106001d3fd3082bf59d0405300160090086e$_sn:1$_ss:0$_pn:5%3Bexp-session$_st:1414196750542$ses_id:1414194483462%3Bexp-session; s_fid=7486C318933E2879-25F3BFA3ED8EC585; s_nr=1414194950805-New
+Connection: keep-alive
+Content-Type: multipart/form-data; boundary=---------------------------38016663816079156891839870716
+Content-Length: 1360
+
+-----------------------------38016663816079156891839870716
+Content-Disposition: form-data; name="t1"
+
+test_1
+-----------------------------38016663816079156891839870716
+Content-Disposition: form-data; name="t2"
+
+test_2
+-----------------------------38016663816079156891839870716
+Content-Disposition: form-data; name="t2"
+
+test_3
+-----------------------------38016663816079156891839870716
+Content-Disposition: form-data; name="f1"; filename="index.slim"
+Content-Type: application/octet-stream
+
+doctype html
+html
+  body
+    form action="http://localhost" method="POST"
+
+      input type="text" name="i1"
+      input type="text" name="i2"
+      input type="file" name="f1"
+      textarea name="t1"
+
+      input type="submit"
+
+-----------------------------38016663816079156891839870716
+Content-Disposition: form-data; name="f2"; filename="index.html"
+Content-Type: text/html
+
+<!DOCTYPE html><html><body><form action="http://localhost" method="POST">
+  <input name="i1" type="text" /><input name="i2" type="text" /><input name="f1" type="file" /><textarea name="t1"></textarea><input type="submit" /></form></body></html>
+
+-----------------------------38016663816079156891839870716
+Content-Disposition: form-data; name="f2"; filename=""
+Content-Type: application/octet-stream
+
+
+-----------------------------38016663816079156891839870716--
diff --git a/http.gemspec b/http.gemspec
@@ -24,6 +24,7 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency "http_parser.rb", "~> 0.6.0"
   gem.add_runtime_dependency "http-form_data", "~> 1.0.0"
+  gem.add_runtime_dependency "rack-cache",     "~> 1.2"
 
   gem.add_development_dependency "bundler", "~> 1.0"
 end
diff --git a/lib/http/cache.rb b/lib/http/cache.rb
@@ -0,0 +1,146 @@
+require "time"
+require "rack-cache"
+
+module HTTP
+  class Cache
+    # NoOp logger.
+    class NullLogger
+      def error(_msg = nil)
+      end
+
+      def debug(_msg = nil)
+      end
+
+      def info(_msg = nil)
+      end
+
+      def warn(_msg = nil)
+      end
+    end
+
+    # @return [Response] a cached response that is valid for the request or
+    #   the result of executing the provided block
+    #
+    # @yield [request, options] on cache miss so that an actual
+    # request can be made
+    def perform(request, options, &request_performer)
+      req = request.caching
+
+      invalidate_cache(req) if req.invalidates_cache?
+
+      get_response(req, options, request_performer)
+    end
+
+    protected
+
+    # @return [Response] the response to the request, either from the
+    # cache or by actually making the request
+    def get_response(req, options, request_performer)
+      cached_resp = cache_lookup(req)
+      return cached_resp if cached_resp && !cached_resp.stale?
+
+      # cache miss
+      logger.debug { "Cache miss for <#{req.uri}>, making request" }
+      actual_req = if cached_resp
+                     req.conditional_on_changes_to(cached_resp)
+                   else
+                     req
+                   end
+      actual_resp = make_request(actual_req, options, request_performer)
+
+      handle_response(cached_resp, actual_resp, req)
+    end
+
+    # @returns [Response] the most useful of the responses after
+    # updating the cache as appropriate
+    def handle_response(cached_resp, actual_resp, req)
+      if actual_resp.status.not_modified? && cached_resp
+        logger.debug { "<#{req.uri}> not modified, using cached version." }
+        cached_resp.validated!(actual_resp)
+        store_in_cache(req, cached_resp)
+        return cached_resp
+
+      elsif req.cacheable? && actual_resp.cacheable?
+        store_in_cache(req, actual_resp)
+        return actual_resp
+
+      else
+        return actual_resp
+      end
+    end
+
+    # @return [HTTP::Response::Caching] the actual response returned
+    # by request_performer
+    def make_request(req, options, request_performer)
+      req.sent_at = Time.now
+
+      request_performer.call(req, options).caching.tap do |res|
+        res.received_at  = Time.now
+        res.requested_at = req.sent_at
+      end
+    end
+
+    # @return [HTTP::Response::Caching, nil] the cached response for the request
+    def cache_lookup(request)
+      return nil if request.skips_cache?
+
+      rack_resp = metastore.lookup(request, entitystore)
+      return if rack_resp.nil?
+
+      HTTP::Response.new(
+        rack_resp.status, "1.1", rack_resp.headers, stringify(rack_resp.body)
+      ).caching
+    end
+
+    # Store response in cache
+    #
+    # @return [nil]
+    #
+    # ---
+    #
+    # We have to convert the response body in to a string body so
+    # that the cache store reading the body will not prevent the
+    # original requester from doing so.
+    def store_in_cache(request, response)
+      response.body = response.body.to_s
+      metastore.store(request, response, entitystore)
+      nil
+    end
+
+    # Invalidate all response from the requested resource
+    #
+    # @return [nil]
+    def invalidate_cache(request)
+      metastore.invalidate(request, entitystore)
+    end
+
+    # Inits a new instance
+    #
+    # @option opts [String] :metastore   URL to the metastore location
+    # @option opts [String] :entitystore URL to the entitystore location
+    # @option opts [Logger] :logger      logger to use
+    def initialize(opts)
+      @metastore   = storage.resolve_metastore_uri(opts.fetch(:metastore))
+      @entitystore = storage.resolve_entitystore_uri(opts.fetch(:entitystore))
+      @logger = opts.fetch(:logger) { NullLogger.new }
+    end
+
+    attr_reader :metastore, :entitystore, :logger
+
+    def storage
+      @@storage ||= Rack::Cache::Storage.new # rubocop:disable Style/ClassVars
+    end
+
+    def stringify(body)
+      if body.respond_to?(:each)
+        "".tap do |buf|
+          body.each do |part|
+            buf << part
+          end
+        end
+      else
+        body.to_s
+      end
+    end
+  end
+end
diff --git a/lib/http/cache/headers.rb b/lib/http/cache/headers.rb
@@ -0,0 +1,100 @@
+require "delegate"
+
+require "http/errors"
+require "http/headers"
+
+module HTTP
+  class Cache
+    # Convenience methods around cache control headers.
+    class Headers < ::SimpleDelegator
+      def initialize(headers)
+        if headers.is_a? HTTP::Headers
+          super headers
+        else
+          super HTTP::Headers.coerce headers
+        end
+      end
+
+      # @return [Boolean] does this message force revalidation
+      def forces_revalidation?
+        must_revalidate? || max_age == 0
+      end
+
+      # @return [Boolean] does the cache control include 'must-revalidate'
+      def must_revalidate?
+        matches?(/\bmust-revalidate\b/i)
+      end
+
+      # @return [Boolean] does the cache control include 'no-cache'
+      def no_cache?
+        matches?(/\bno-cache\b/i)
+      end
+
+      # @return [Boolean] does the cache control include 'no-stor'
+      def no_store?
+        matches?(/\bno-store\b/i)
+      end
+
+      # @return [Boolean] does the cache control include 'public'
+      def public?
+        matches?(/\bpublic\b/i)
+      end
+
+      # @return [Boolean] does the cache control include 'private'
+      def private?
+        matches?(/\bprivate\b/i)
+      end
+
+      # @return [Numeric] the max number of seconds this message is
+      # considered fresh.
+      def max_age
+        explicit_max_age || seconds_til_expires || Float::INFINITY
+      end
+
+      # @return [Boolean] is the vary header set to '*'
+      def vary_star?
+        get("Vary").any? { |v| "*" == v.strip }
+      end
+
+      private
+
+      # @return [Boolean] true when cache-control header matches the pattern
+      def matches?(pattern)
+        get("Cache-Control").any? { |v| v =~ pattern }
+      end
+
+      # @return [Numeric] number of seconds until the time in the
+      # expires header is reached.
+      #
+      # ---
+      # Some servers send a "Expire: -1" header which must be treated as expired
+      def seconds_til_expires
+        get("Expires")
+          .map { |e| http_date_to_ttl(e) }
+          .max
+      end
+
+      def http_date_to_ttl(t_str)
+        ttl = to_time_or_epoch(t_str) - Time.now
+
+        ttl < 0 ? 0 : ttl
+      end
+
+      # @return [Time] parses t_str at a time; if that fails returns epoch time
+      def to_time_or_epoch(t_str)
+        Time.httpdate(t_str)
+      rescue ArgumentError
+        Time.at(0)
+      end
+
+      # @return [Numeric] the value of the max-age component of cache control
+      def explicit_max_age
+        get("Cache-Control")
+          .map { |v| (/max-age=(\d+)/i).match(v) }
+          .compact
+          .map { |m| m[1].to_i }
+          .max
+      end
+    end
+  end
+end
diff --git a/lib/http/cache/null_cache.rb b/lib/http/cache/null_cache.rb
@@ -0,0 +1,13 @@
+module HTTP
+  class Cache
+    # NoOp cache. Always makes the request. Allows avoiding
+    # conditionals in the request flow.
+    class NullCache
+      # @return [Response] the result of the provided block
+      # @yield [request, options] so that the request can actually be made
+      def perform(request, options)
+        yield(request, options)
+      end
+    end
+  end
+end
diff --git a/lib/http/chainable.rb b/lib/http/chainable.rb
@@ -107,6 +107,10 @@ def follow(opts = true)
     # @see #follow
     alias_method :with_follow, :follow
 
+    def with_cache(cache)
+      branch default_options.with_cache(cache)
+    end
+
     # Make a request with the given headers
     # @param headers
     def with_headers(headers)

diff --git a/lib/http/client.rb b/lib/http/client.rb
@@ -42,6 +42,12 @@ def request(verb, uri, opts = {})
 
     # Perform a single (no follow) HTTP request
     def perform(req, options)
+      options.cache.perform(req, options) do |r, opts|
+        make_request(r, opts)
+      end
+    end
+
+    def make_request(req, options)
       # finish previous response if client was re-used
       # TODO: this is pretty wrong, as socket shoud be part of response
       #       connection, so that re-use of client will not break multiple

diff --git a/lib/http/errors.rb b/lib/http/errors.rb
@@ -11,6 +11,9 @@ class ResponseError < Error; end
   # Requested to do something when we're in the wrong state
   class StateError < ResponseError; end
 
+  # Generic Cache error
+  class CacheError < Error; end
+
   # Header name is invalid
   class InvalidHeaderNameError < Error; end
 end