Nginx, proxy_cache and badly behaving app (wordpress)

nginxreverse-proxyWordpress

I've been fighting with configuring Nginx as a reverse proxy.
I've got most of the mechanics working, but I've been fighting for the last 3 days to control cache headers and being newbie to nginx this is driving me crazy…

It looks like I'm not getting how location block chain up.

I would really appreciate a bit of help

What I would like is simple

  • all content-type text/html to have header Cache-Control: max-age=60, public, must-revalidate
  • all css/js/images etc… to have header Cache-Control: max-age=315360000, public
  • 404 and all errors not to have cache control

Thanks for any advice on how this should be done.

here I have the proxy settings (seems to work as expected)

proxy_cache_path /tmp/example levels=1:2 keys_zone=example:100m max_size=4g inactive=60m;
proxy_temp_path /tmp/example/tmp;

server {
  listen 80;
  server_name example.com;

  # is this realy needed?
  # Perhaps it should point to an empty folder
  root /var/www/vhosts/example.com/www;

  location ~ /\. {
    deny all;
  }

  location ~ /purge(/.*) {
    proxy_cache_purge nx_anto "$scheme$request_method$host$1";
  }

  location / {

    proxy_cache example;
    proxy_cache_key "$scheme$request_method$host$request_uri";
    proxy_connect_timeout 60s;

    proxy_cache_methods GET HEAD;

    # don't honour cache headers from the app server
    # proxy_ignore_headers Cache-Control Set-Cookie Expires Cache-Control;

    proxy_cache_lock on;
    # proxy_cache_min_uses 3;

    # proxy_cache_valid 301       24h;
    # keep objects long enough for proxy_cache_use_stale
    proxy_cache_valid 200 302     1h;    
    # 404 errors
    proxy_cache_valid any       5m;


    proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;

    # to pass If-Modified-Since to the origin server
    # proxy_cache_revalidate on;

    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header X-Forwarded-Proto $scheme;

    # hide cache-related headers
    proxy_hide_header X-Powered-By;

    # this avoids having duplicate Vary headers sent to final client.
    proxy_hide_header Vary;
    # proxy_hide_header Pragma;
    # proxy_hide_header Expires;
    # proxy_hide_header Last-Modified;
    # proxy_hide_header Cache-Control;
    # proxy_hide_header Set-Cookie;

    set $skip_cache 0;
    # POST requests and urls with a query string should always go to PHP
    if ($request_method = POST) {
      set $skip_cache 1;
    }
    # wordpress adds query strings to css and js that we want to cache
    #   if ($query_string != "") {
    #     set $skip_cache 1;
    #   }

    # Don't cache uris containing the following segments
    if ($request_uri ~* "/wp-admin/|/xmlrpc.php|wp-.*.php|/feed/|index.php|sitemap(_index)?.xml") {
        set $skip_cache 1;
    }

    # Don't use the cache for logged in users or recent commenters
    # if ($http_cookie ~* "comment_author|wordpress_[a-f0-9]+|wp-postpass|wordpress_no_cache|wordpress_logged_in") {
    # wordpress_[a-f0-9]+ was blocking cache on loged out users
    if ($http_cookie ~* "comment_author|wp-postpass|wordpress_no_cache|wordpress_logged_in") {
        set $skip_cache 1;
    }

    proxy_cache_bypass $skip_cache;
    proxy_no_cache $skip_cache;

    # for debugging
    add_header "X-Cache-Status" $upstream_cache_status;
    add_header "X-Dummy" $sent_http_content_type;

    proxy_pass http://example.com:8080;
  }

}

now comes the app vhost

server {
  listen 8080;
  server_name example.com;
  root /var/www/vhosts/example.com/www;

  index index.php;

  server_tokens off;
  etag off;

  location / {
    try_files $uri $uri/ /index.php?$args;
  }

  location ~ .php$ {

    include fastcgi_params;
    fastcgi_param PATH_TRANSLATED $document_root$fastcgi_script_name;
    fastcgi_pass unix:/run/php/php5.6-fpm_example.sock;
    access_log /var/log/nginx/phpfpmonly-access-example.log;

    try_files $uri /index.php =404;

## I never got this condition to work
#     if ($sent_http_content_type ~* "text/html") {
#      add_header "Cache-Control" "public, must-revalidate, proxy-revalidate";
#      expires 60s;
#     }
  }
}

now I would like to control headers for css/js etc…

location ~* \.(ogg|ogv|svg|svgz|eot|otf|woff|mp4|ttf|rss|atom||zip|tgz|gz|rar|bz2|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf|css|js|ico|gif|jpe?g|png|svg|eot|otf|woff|woff2|ttf|ogg) {
  expires max;
  add_header Cache-Control "public";
}

I've tried putting this location block in any server block (proxy or app) and never got what I wanted.
Placed in the app server block this break php (returning me php source code)

Best Answer

You should be using fastcgi cache, not the proxy cache, unless you have a really good reason to be using a proxy cache. I have a tutorial that addresses exactly what you're trying to do, you can read it here, and it has downloadable configuration files.

You have to have mod_headers built into Nginx to control headers. My tutorial covers that.

SF prefers the answer to be in the question in case websites disappear. The website will be easier to read, and should be around for a good while. Copied below for reference.

Nginx configuration file

# Caching. Putting the cache into /dev/shm keeps it in RAM, limited to 10MB, for one day.
# You can move to disk if you like, or extend the caching time
fastcgi_cache_path /dev/shm/hr_nginxcache levels=1:2 keys_zone=HR_CACHE:50m inactive=1440m; #RAM

upstream php {
   server 127.0.0.1:9001;
}


# http production headphone reviews server
server {
  server_name www.example.com;
  listen 443 ssl http2;

  ssl_certificate /var/lib/acme/certs/***CERT_DIRECTORY/fullchain;
  ssl_certificate_key /var/lib/acme/certs/***CERT_DIRECTORY/privkey;

  # Set up preferred protocols and ciphers. TLS1.2 is required for HTTP/2
  ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
  ssl_prefer_server_ciphers on;
  ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DH+3DES:!ADH:!AECDH:!MD5;

  # This tells the browser not to bother trying to use http for an hour - it should probably
  # be put up to a week or so, and leave it disabled for testing
  # add_header Strict-Transport-Security "max-age=3600" always;
  # This does the same but for subdomains as well
  # add_header Strict-Transport-Security "max-age=3600; includeSubDomains" always;

  root /var/www/***folder;

  # First line is a cached access log, second logs immediately
  access_log  /var/log/nginx/hr.access.log main buffer=128k flush=60 if=$log_ua;
  # access_log  /var/log/nginx/hr.access.log main;

  # Rules to work out when cache should/shouldn't be used
  set $skip_cache 0;

  # POST requests and urls with a query string should always go to PHP
  if ($request_method = POST) {
      set $skip_cache 1;
  }   
  if ($query_string != "") {
    set $skip_cache 1;
  }   
  # Don't cache uris containing the following segments. 'admin' is for one of my websites, it's not required
  # for everyone. I've removed index.php as I want pages cached.
  #if ($request_uri ~* "/wp-admin/|/admin-*|/purge*|/xmlrpc.php|wp-.*.php|/feed/|index.php|sitemap(_index)?.xml") {
  if ($request_uri ~* "/wp-admin/|/admin-*|/purge*|/xmlrpc.php|wp-.*.php|/feed/|sitemap(_index)?.xml") {
    set $skip_cache 1;
  }   
  # Don't use the cache for logged in users or recent commenters
  #  if ($http_cookie ~* "comment_author|wordpress_[a-f0-9]+|wp-postpass|wordpress_no_cache|wordpress_logged_in|code|PHPSESSID") {
  if ($http_cookie ~* "comment_author|wordpress_[a-f0-9]+|wordpress_logged_in|code") {
    set $skip_cache 1;
  }

  # If we skip the cache it's likely customised for one user. Set the caching headers to match.
  # http://www.mobify.com/blog/beginners-guide-to-http-cache-headers/
  if ($skip_cache = 1) {
    set $cacheControl "private, max-age=0, s-maxage=0, no-cache, no-store";
  }
  if ($skip_cache = 0) {
    set $cacheControl "public, max-age=86400, s-maxage=86400";
  }

  # Default location to serve
  location / {
    # If the file can't be found try adding a slash on the end - it might be
    # a directory the client is looking for. Then try the Wordpress blog URL
    # this might send a few requests to PHP that don't need to go that way
    try_files $uri $uri/ /blog/index.php?$args;
    more_clear_headers Server; more_clear_headers "Pragma"; more_clear_headers "Expires";
    # add_header Z_LOCATION "hr_root"; add_header URI $uri; # DEBUG
  }

  # Add trailing slash to */wp-admin requests.
  rewrite /blog/wp-admin$ $scheme://$host$uri/ permanent;

  # HR SEO rewrite rules
  location /headphone {
    rewrite ^/headphone\/([0-9a-zA-Z_\-\s\+]+)\/([0-9a-zA-Z_\-\s\+\(\)\.]+)$ /headphone.php?action=searchOne&headphoneName=$2&manufacturerName=$1;
    # add_header Z_LOCATION "headphone-rewrite"; add_header URI $uri; # DEBUG
  }
  location /headphones {
    rewrite ^/headphones\/([0-9a-zA-Z_\-\s\+]+)$ /headphone.php?action=searchManufacturer&manufacturerName=$1;
    # add_header Z_LOCATION "headphoneS-rewrite"; add_header URI $uri; # DEBUG
  }

  # Don't log robots errors but log access
  location = /robots.txt {
    allow all; log_not_found off; 
    # on is the default - access_log on;
    more_clear_headers Server; more_clear_headers "Pragma";
  }

  #Deny public access to wp-config.php
  location ~* wp-config.php {
    deny all;
  }

  # Don't log errors finding static resources, and optionally set the expires time to maximum
  # NB I removed ICO so I could redirect favicon below - longer string therefore location matched
  location ~*  \.(jpg|jpeg|png|gif|css|js|ico|svg)$ { 
    log_not_found off; access_log off;
    valid_referers none blocked server_names ~($host) ~(googleusercontent|google|bing|yahoo);
    if ($invalid_referer) {
      rewrite (.*) /stop-stealing-images.png redirect;
      # drop the 'redirect' flag for redirect without URL change (internal rewrite)
    }

    # Set up caching - 8 days for static resources
    # Remove the old unnecessary Pragma and hide the server version
    more_clear_headers "Cache-Control";
    add_header Cache-Control "public, max-age=691200, s-maxage=691200";
    more_clear_headers Server; more_clear_headers "Pragma"; more_clear_headers "Expires";

    # Debug remove
    # add_header Z_LOCATION "HR STATIC RESOURCES REGEX"; add_header URI $uri; 
  }
  # *** Find yourself a suitable graphic
  location = /stop-stealing-images.png { }

  # Rate limit wp-login.php to help prevent brute force attacks
  location = /blog/wp-login.php {
    # Next line applies the rate limit defined above
    limit_req zone=login burst=3;       
    fastcgi_keep_conn on;
    fastcgi_intercept_errors on;
    fastcgi_pass   php;
    include        fastcgi_params;
    fastcgi_param  SCRIPT_FILENAME $document_root$fastcgi_script_name;
    more_clear_headers "Cache-Control";
    more_clear_headers Server; more_clear_headers "Pragma"; more_clear_headers "Expires";

    # No caching
    more_clear_headers "Cache-Control";
    add_header Cache-Control "private, max-age=0, no-cache, no-store";
    more_clear_headers "Expires";

    # DEBUG remove
    # add_header Z_LOCATION "HR-WP-LOGIN"; add_header URI $uri;
    # add_header Z_CACHE_CONTROL $cacheControl;
  }

  # Wordpress admin caching headers are set correctly, for pages and resources. The only reason we define
  # this block separately is to avoid messing with the headers in the main php block.
  # This is probably unnecessary because of the skip_cache variable and may be removed
  location ~* wp-admin {
    fastcgi_keep_conn on;
    fastcgi_intercept_errors on;
    fastcgi_pass   php;
    include        fastcgi_params;
    fastcgi_param  SCRIPT_FILENAME $document_root$fastcgi_script_name;
    # add_header Z_LOCATION "WP_ADMIN"; add_header URI $uri; add_header "Z_SKIP_CACHE" $skip_cache; # DEBUG
  }

  # Send HipHop and PHP requests to HHVM
  location ~ \.(hh|php)$ {
    fastcgi_keep_conn on;
    fastcgi_intercept_errors on;
    fastcgi_pass   php;
    include        fastcgi_params;
    fastcgi_param  SCRIPT_FILENAME $document_root$fastcgi_script_name;

    # Use the cache defined above. Cache 200 (success) status's, for 24 hours, and cache
    # specific other status's for an hour. This helps mitigate DDOS attacks.
    # Only cache GET and HEAD requests
    fastcgi_cache HR_CACHE;
    fastcgi_cache_valid 200 1440m;
    fastcgi_cache_valid 403 404 405 410 414 301 302 307 60m;
    add_header X-Cache $upstream_cache_status;

    fastcgi_cache_methods GET HEAD; 
    fastcgi_cache_bypass $skip_cache;
    fastcgi_no_cache $skip_cache;

    # Set the cache control headers we prepared earlier. Remove the old unnecessary Pragma and hide
    # the server version. Clearing existing headers seems necessary
    more_clear_headers "Cache-Control";
    add_header Cache-Control $cacheControl;
    more_clear_headers "Pragma"; more_clear_headers Server; more_clear_headers "Expires";

    # add_header Z_LOCATION "HR PHP MAIN"; add_header URI $uri;
  }

  # Deny access to uploads which aren’t images, videos, music, etc.
  location ~* ^/wp-content/uploads/.*.(html|htm|shtml|php|js|swf)$ {
    deny all;
    # add_header Z_LOCATION "DENY WPCONTENT UPLOADS"; add_header URI $uri; # DEBUG
  }

  # Create a custom error page that gives the user a more useful error message
  error_page 400 404 500 502 503 504 /error.html;
  location = /error.html {
    root /var/www/hr;
    internal;
  }

  # This is for issuing certificates
  location /.well-known/acme-challenge/ {
    root /var/www/acme-challenge/;
  }

}

# Forward non-www requests to www
server {
    listen       80;
    server_name  example.com www.example.com;
    access_log  /var/log/nginx/hr.access.log main buffer=128k flush=1m if=$log_ua;
    return       301 https://www.example.com$request_uri;
}

server {
  listen 443 ssl http2;
  server_name example.com;

  ssl_certificate /var/lib/acme/certs/***CERT_DIRECTORY/fullchain;
  ssl_certificate_key /var/lib/acme/certs//***CERT_DIRECTORY/privkey;

  # Set up preferred protocols and ciphers. TLS1.2 is required for HTTP/2
  ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
  ssl_prefer_server_ciphers on;
  ssl_ciphers ECDH+AESGCM:ECDH+AES256:ECDH+AES128:DH+3DES:!ADH:!AECDH:!MD5;

  access_log  /var/log/nginx/hr.access.log main buffer=128k flush=1m if=$log_ua;

  return 301 https://www.example.com$request_uri;

Here's how I built Nginx

cd /home/ec2-user
mkdir nginx-build
cd nginx-build
service nginx stop
yum groupinstall "Development Tools"
yum install pcre-devel zlib-devel openssl-devel
wget http://nginx.org/download/nginx-1.9.11.tar.gz
wget http://labs.frickle.com/files/ngx_cache_purge-2.3.tar.gz
wget https://github.com/openresty/headers-more-nginx-module/archive/v0.29.tar.gz
tar -xzf nginx-1.9.11.tar.gz
tar -xzf ngx_cache_purge-2.3.tar.gz
tar -xzf v0.29.tar.gz
tar -xzf 1.9.32.10.tar.gz    # Google Pagespeed, optional
ngx_version=1.9.32.10
wget https://github.com/pagespeed/ngx_pagespeed/archive/release-${ngx_version}-beta.zip   # Google Pagespeed, optional
cd ngx_pagespeed-release-1.9.32.10-beta   # Google Pagespeed, optional
wget https://dl.google.com/dl/page-speed/psol/${ngx_version}.tar.gz   # Google Pagespeed, optional
cd ../nginx-1.9.9
# Note that I have no idea what the next line does but it was in the official guide
PS_NGX_EXTRA_FLAGS="--with-cc=/opt/rh/devtoolset-2/root/usr/bin/gcc"
# Safe option, slower, lots of modules included
#./configure --prefix=/etc/nginx --sbin-path=/usr/sbin/nginx --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --pid-path=/var/run/nginx.pid --lock-path=/var/run/nginx.lock --http-client-body-temp-path=/var/cache/nginx/client_temp --http-proxy-temp-path=/var/cache/nginx/proxy_temp --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp --http-scgi-temp-path=/var/cache/nginx/scgi_temp --user=nginx --group=nginx --with-http_ssl_module --with-http_realip_module --with-http_addition_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_random_index_module --with-http_secure_link_module --with-http_stub_status_module --with-http_auth_request_module --with-threads --with-stream --with-stream_ssl_module --with-http_slice_module --with-mail --with-mail_ssl_module --with-file-aio --with-ipv6 --with-http_v2_module --with-cc-opt='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic' --add-module=/tmp/ngx_cache_purge-2.3 --add-module=/tmp/headers-more-nginx-module-0.29 --with-http_realip_module --add-modeule=../ngx_pagespeed-release-1.9.32.10-beta
# Many plugins removed, extra optimisations including some JUST for the machine it's compiled on
./configure --prefix=/etc/nginx --sbin-path=/usr/sbin/nginx --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --pid-path=/var/run/nginx.pid --lock-path=/var/run/nginx.lock --http-client-body-temp-path=/var/cache/nginx/client_temp --http-proxy-temp-path=/var/cache/nginx/proxy_temp --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp --http-scgi-temp-path=/var/cache/nginx/scgi_temp --user=nginx --group=nginx --with-http_ssl_module --with-http_realip_module --with-http_gunzip_module --with-http_gzip_static_module --with-threads --with-file-aio --with-ipv6 --with-http_v2_module --with-cc-opt='-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=native' --add-module=../ngx_cache_purge-2.3 --add-module=../headers-more-nginx-module-0.29 --with-http_realip_module --add-module=../ngx_pagespeed-release-1.9.32.10-beta
make && make install
make clean  (NB: optional)
service nginx start
Related Topic