Nginx – Slow Nginx + PHP-FPM during High Traffic on Magento Enterprise

We are using Nginx with PHP-FPM for Magento Enterprise Edition 1.12

Everything works fine but when we get visitors over 150 the speed drops. During this time we have noticed that CPU utilization is 10% with over 40% free mem.

Server Configuration:

CPU 2 x Intel Xeon E5-2680 2.7 GHz 8 Cores 16 Threads 8 GT/s (w H/T) 
RAM 8GB x 8 = 64 GB Total 
NIC 1G connected to Firewall
NIC 1G connected to DB server running percona MySQL

Configuration: nginx.conf

user nginx;
worker_processes 32;

error_log  /var/log/nginx/error.log;

pid        /var/run/nginx.pid;

events {
worker_connections  1024;
# multi_accept on;
}

http {
include       mime.types;
default_type  application/octet-stream;

log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                  '$status $body_bytes_sent "$http_referer" '
                  '"$http_user_agent" "$http_x_forwarded_for"';
access_log  /var/log/nginx/access.log  main;

sendfile          on;
tcp_nopush        on;
tcp_nodelay       off;
autoindex         off;
keepalive_timeout 10;

## detect https
map $scheme $fastcgi_https {
    default off;
    https on;
}
# Store FastCGI Cached (microcaching) of PHP pages in our tmpfs ramdisk
fastcgi_cache_path /tmpfs/nginx levels=1:2 keys_zone=mage:5m max_size=1g inactive=1h;
open_file_cache max=10000 inactive=30m;
open_file_cache_valid 10m;
open_file_cache_min_uses 2;
open_file_cache_errors on;
proxy_read_timeout 10;

upstream phpfpm {
    server localhost:9000;
}

gzip on;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_proxied any;
gzip_min_length  1100;
gzip_buffers 16 8k;
gzip_types text/plain text/css application/x-javascript text/xml application/xml
application/xml+rss text/javascript;
# Some version of IE 6 don't handle compression well on some mime-types, so just disable for them
gzip_disable "MSIE [1-6].(?!.*SV1)";
# Set a vary header so downstream proxies don't send cached gzipped content to IE6
gzip_vary on;

# Load config files from the /etc/nginx/conf.d directory
include /etc/nginx/conf.d/*.conf;

# Health-check server
server {
    listen       80;
    server_name  localhost;
    charset      utf-8;
    access_log off;
    location / {
        root   share/nginx/html;
        index  index.html index.htm;

    }

    #error_page  404              /404.html;
    # redirect server error pages to the static page /50x.html
    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   share/nginx/html;
    }

    }

}

Configurtation PHP-FPM.conf

[global]
pid = /var/run/php-fpm/php-fpm.pid
error_log = /var/log/php-fpm/error.log
[www]
listen = 127.0.0.1:9000
listen.backlog = 65536
listen.allowed_clients = 127.0.0.1
user = nginx
group = nginx
request_terminate_timeout = 600
pm = static
pm.max_children = 846
#pm.start_servers = 200
#pm.min_spare_servers = 200
#pm.max_spare_servers = 200
pm.max_requests = 10000
slowlog = /var/log/php-fpm/www-slow.log

configuation fastcgi.conf

#fastcgi_set_header "Set-Cookie";
#if ($http_cookie != "X-Store=1" ) {
#  add_header Set-Cookie "X-Store=1; path=/";
#}

## Catch 404s that try_files miss
if (!-e $request_filename) { rewrite / /index.php last; }

## Blacklist media location from fcgi
if ($uri !~ "^/(media|skin|js)/") {
fastcgi_pass phpfpm;
}

## Tell the upstream who is making the request
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_redirect off;

# Ensure PHP knows when we use HTTPS
fastcgi_param  HTTPS           $fastcgi_https;

## Fcgi Settings
include                        fastcgi_params;
fastcgi_connect_timeout        60;
fastcgi_send_timeout           60;
fastcgi_read_timeout           300;
fastcgi_buffer_size            4k;
fastcgi_buffers 512            4k;
fastcgi_busy_buffers_size      8k;
fastcgi_temp_file_write_size   256k;
fastcgi_intercept_errors       off;
fastcgi_index  index.php;
fastcgi_param  SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_param  SCRIPT_NAME     $fastcgi_script_name;
# nginx will buffer objects to disk that are too large for the buffers above
fastcgi_temp_path              /tmpfs/nginx/tmp 1 2;
#fastcgi_keep_conn              on; # NGINX 1.1.14
expires                        off; ## Do not cache dynamic content

Top

top - 13:24:45 up  5:58,  1 user,  load average: 0.88, 0.70, 0.71
Tasks: 1465 total,   3 running, 1462 sleeping,   0 stopped,   0 zombie
Cpu(s):  4.1%us,  0.1%sy,  0.0%ni, 95.8%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
Mem:  65942708k total, 38795452k used, 27147256k free,   213844k buffers
Swap:  8388600k total,        0k used,  8388600k free,  1263300k cached

PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
3300 nginx     20   0 1404m 169m 127m S 28.3  0.3   0:30.33 php-fpm
4165 nginx     20   0 1401m 156m 117m S 26.0  0.2   0:30.17 php-fpm
4072 nginx     20   0 1410m 163m 118m S 22.7  0.3   0:27.75 php-fpm
3710 nginx     20   0 1402m 163m 123m R 12.8  0.3   0:32.39 php-fpm
3578 nginx     20   0 1401m 157m 118m S 11.9  0.2   0:27.39 php-fpm

We are using magento enterprise and using Full Page Cache. APC is also installed and active.

Problem: Get extremely slow during high traffic.

Steve