Nginx : add_header directive not working

My Issue:
nginx add_header directive doesn't appear to be working

What I have tried:
In my nginx conf I have this:

location ~* \.(ttf|woff|eot|otf|woff2|svg|svgz)$ {
        access_log /var/log/nginx/fonts.access.log;
        add_header Access-Control-Allow-Origin *; expires 1M;
}

When I request a font resource like so:

curl -i -s -D -  -XGET http://my.server.com/assets/my_font-f748f9b5f469637888371ef2a5a81765.eot -o /dev/null
HTTP/1.1 200 OK
Accept-Ranges: bytes
Cache-Control: max-age=2592000
Content-Type: application/octet-stream
Date: Tue, 08 Sep 2015 16:42:55 GMT
ETag: "55eee9fb-d980"
Expires: Thu, 08 Oct 2015 16:42:55 GMT
Last-Modified: Tue, 08 Sep 2015 14:00:27 GMT
Server: nginx/1.4.6 (Ubuntu)
Content-Length: 55680
Connection: keep-alive

Notice I do not get back the Access-Control-Allow-Origin * header.
To confirm my nginx is returning from that location block I added location block logging. I do see the request for fonts being made in my font.access.log.

$ tail -1 fonts.access.log
172.31.27.203 - - [08/Sep/2015:16:42:55 +0000] "GET /assets/my_font-f748f9b5f469637888371ef2a5a81765.eot HTTP/1.1" 200 55680 "-" "curl/7.30.0"

Other Information:
Nginx version and compilitation flags:

$ nginx -V
nginx version: nginx/1.4.6 (Ubuntu)
built by gcc 4.8.2 (Ubuntu 4.8.2-19ubuntu1)
TLS SNI support enabled
configure arguments: 
--with-cc-opt='-g -O2 -fstack-protector 
--param=ssp-buffer-size=4 -Wformat -Werror=format-security -D_FORTIFY_SOURCE=2' 
--with-ld-opt='-Wl,-Bsymbolic-functions -Wl,-z,relro' 
--prefix=/usr/share/nginx 
--conf-path=/etc/nginx/nginx.conf 
--http-log-path=/var/log/nginx/access.log 
--error-log-path=/var/log/nginx/error.log 
--lock-path=/var/lock/nginx.lock 
--pid-path=/run/nginx.pid 
--http-client-body-temp-path=/var/lib/nginx/body 
--http-fastcgi-temp-path=/var/lib/nginx/fastcgi 
--http-proxy-temp-path=/var/lib/nginx/proxy 
--http-scgi-temp-path=/var/lib/nginx/scgi 
--http-uwsgi-temp-path=/var/lib/nginx/uwsgi 
--with-debug 
--with-pcre-jit 
--with-ipv6 
--with-http_ssl_module 
--with-http_stub_status_module 
--with-http_realip_module 
--with-http_addition_module 
--with-http_dav_module 
--with-http_geoip_module 
--with-http_gzip_static_module 
--with-http_image_filter_module 
--with-http_spdy_module 
--with-http_sub_module 
--with-http_xslt_module 
--with-mail 
--with-mail_ssl_module

My conf is good:

sudo nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful

My sites conf:

real_ip_header X-Forwarded-For;
log_format mysite_log_fmt '[$time_local][$status][$request] from="$remote_addr" host="$host" ua="$http_user_agent"';

upstream unicorn_mysite {
 server unix:/srv/www/mysite/shared/sockets/unicorn.sock fail_timeout=0;
}

server {
  listen 80;
  server_name
  <ip>
  <other ip>
  <domain name>
  <elb domain name>

  access_log /var/log/nginx/mysites.access.log mysite_log_fmt;

  keepalive_timeout 5;

  root /srv/www/mysite/current/public/;

  location ~* \.(ttf|woff|eot|otf|woff2|svg|svgz)$ {
        access_log /var/log/nginx/fonts.access.log;
        add_header Access-Control-Allow-Origin "*";
  }

  location / {
    try_files $uri/index.html $uri/index.htm @unicorn;
    access_log /var/log/nginx/slash.access.log;
  }

  location @unicorn {
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_set_header Host $http_host;
    proxy_redirect off;
    access_log /var/log/nginx/unicorn.access.log;

    proxy_read_timeout 60;
    proxy_send_timeout 60;

    # If you don't find the filename in the static files
    # Then request it from the unicorn server
    if (!-f $request_filename) {
      proxy_pass http://unicorn_mysite;
      break;
    }
  }

  location /nginx_status {
    stub_status on;
    access_log off;
    allow 127.0.0.1;
    deny all;
  }

  error_page 500 502 503 504 /500.html;
  location = /500.html {
    root /srv/www/mysite/current/public/;
  }
}