Use Varnish Cache only on-disk

I don't think that there is a way to explicitly invalidate cached items, but here is an example of how to do the rest. Update: As mentioned by Piotr in another answer, there is a cache purge module that you can use. You can also force a refresh of a cached item using nginx's proxy_cache_bypass - see Cherian's answer for more information.

In this configuration, items that aren't cached will be retrieved from example.net and stored. The cached versions will be served up to future clients until they are no longer valid (60 minutes).

Your Cache-Control and Expires HTTP headers will be honored, so if you want to explicitly set an expiration date, you can do that by setting the correct headers in whatever you are proxying to.

There are lots of parameters that you can tune - see the nginx Proxy module documentation for more information about all of this including details on the meaning of the different settings/parameters: http://nginx.org/r/proxy_cache_path

http {
  proxy_cache_path  /var/www/cache levels=1:2 keys_zone=my-cache:8m max_size=1000m inactive=600m;
  proxy_temp_path /var/www/cache/tmp; 


  server {
    location / {
      proxy_pass http://example.net;
      proxy_cache my-cache;
      proxy_cache_valid  200 302  60m;
      proxy_cache_valid  404      1m;
    }
  }
}

Ok, in the end I managed to solve this using the following vcl file. Note that I added a couple of extra bits to allow cache expiration grace when the backend has died.

It seems my main failure was using unset req.http.Set-Cookie; when I should have been using unset obj.http.Set-Cookie; in the vcl_fetch section. (obj in vcl_fetch and req in vcl_recv section).

director rails_director round-robin {
  { 
    .backend = { 
      .host = "xxx.xxx.xxx.xxx"; 
      .port = "http";
      .probe = {
        .url = "/lbcheck/lbuptest";
        .timeout = 0.3 s;
        .window = 8;
        .threshold = 3;
      }
    } 
  }
}

sub vcl_recv {

  if (req.backend.healthy) {
    set req.grace = 30s;
  } else {
    set req.grace = 1h;
  }

  if (req.url ~ "^/login") {
    pipe;
  }

  if (req.url ~ "^/sessions") {
    pipe;
  }

  if (req.url ~ "\.(css|js|jpg|jpeg|gif|ico|png)\??\d*$") {
    unset req.http.cookie;
    lookup;
  } else {
    if (req.http.cookie ~ "user_credentials") {
      pipe;
    } else {
      unset req.http.cookie;
    }
  }

  # Only cache GET and HEAD requests
  if (req.request != "GET" && req.request != "HEAD") {
    pipe;
  }

}

sub vcl_fetch {

  set obj.grace = 1h;

  if (req.url ~ "^/login") {
    pass;
  }

  if (req.url ~ "^/sessions") {
    pass;
  }

  if (req.http.cookie ~ "user_credentials") {
    pass;
  } else {
    unset obj.http.Set-Cookie;
  }

  # cache CSS and JS files
  if (req.url ~ "\.(css|js|jpg|jpeg|gif|ico|png)\??\d*$") {
    unset obj.http.Set-Cookie;
  } 

  if (obj.status >=400 && obj.status <500) {
    error 404 "File not found";
  }

  if (obj.status >=500 && obj.status <600) {
    error 503 "File is Temporarily Unavailable";
  }

}

sub vcl_deliver {
  if (obj.hits > 0) {
          set resp.http.X-Cache = "HIT";
  } else {
          set resp.http.X-Cache = "MISS";
  }
}