Nginx + Unicorn Rails Application gets 404 when requests number are high

http-status-code-404nginxruby-on-railsUbuntuunicorn

I have a Rails application deployed using Nginx + Unicorn. Last night, when the traffic peaked, Nginx was getting lots of 404 errors. When I dig into logs, those 404 requests on the nginx side did not even get to Rails because I could not find these requests in Rails log. What is the problem here? Rails app was overloaded?

The server is a dual-core 4GB memory virtual host with Ubuntu 11.10 + nginx 1.0.5, the CPU was pretty much packed during the peak traffic. I am using 4 unicorn worker processes at this moment.

Thanks!

nginx.conf

user deployer staff;
worker_processes 4;
pid /var/run/nginx.pid;

events {
    worker_connections 1024;
    multi_accept on;
}

http {
    sendfile on;
    tcp_nopush on;
    tcp_nodelay off;
    keepalive_timeout 65;
    types_hash_max_size 2048;
    underscores_in_headers on;

    include /etc/nginx/mime.types;
    default_type application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                  '$status $body_bytes_sent "$http_referer" '
                  '"$http_user_agent" "$http_x_forwarded_for"';

    access_log /var/log/nginx/access.log main;
    error_log /var/log/nginx/error.log warn;

    gzip on;
    gzip_disable "msie6";

    gzip_proxied any;
    gzip_min_length 500;
    gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;

##
# Virtual Host Configs
##

    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;
}

upstream
upstream app {
server unix:/tmp/app_production.socket fail_timeout=0;
}

server {
    client_max_body_size 10m;
    listen       80;
    server_name  xxx.xxx.xxx.xxx;

    #charset koi8-r;
    root /home/deployer/apps/app_production/current/public;
    try_files $uri /system/maintenance.html @app;
    #access_log  /var/log/nginx/log/host.access.log  main;

    location @app {
        proxy_pass http://app;
        proxy_redirect off;

        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

        client_max_body_size       10m;
        client_body_buffer_size    128k;

        proxy_connect_timeout      90;
        proxy_send_timeout         90;
        proxy_read_timeout         90;

        proxy_buffer_size          4k;
        proxy_buffers              4 32k;
        proxy_busy_buffers_size    64k;
        proxy_temp_file_write_size 64k;
    }

    #error_page  404              /404.html;

    # redirect server error pages to the static page /50x.html
    #
    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   /usr/share/nginx/html;
    }

    # static resource
    location ~ ^/(assets|images|javascripts|stylesheeets|system)/ {
        gzip_static on;
        expires max;
        add_header Cache-Control public;
        add_header Last-Modified "";
        add_header ETag "";

        open_file_cache max=100 inactive=500s;
        open_file_cache_valid 600s;
        open_file_cache_errors on;
        break;
    }
}

And most of the 404 errors in nginx log looks like this

2013/02/21 06:15:40 [error] 1953#0: *7081 connect() to unix:/tmp/app_production.socket failed (11: Resource temporarily unavailable) while connecting to upstream, client: xxx.xxx.xxx.xxx, server: xxx.xxx.xxx.xxx, request: "GET /api/modules HTTP/1.1", upstream: "http://unix:/tmp/app_production.socket:/api/modules", host: "myapp.com"

Best Answer

You did probably hit the maximum number of open sockets, try increasing worker_rlimit_nofile in nginx and fs.file-max in OS.