Nginx proxy: deny all apart from sub-path

I'm trying to setup an Nginx proxy for a node app that is currently under development.

I'm trying to allow only whitelisted IPs to access the main site, but I have an /api path that I would like to be accessible by any IP.

I've tried defining the location block in different orders, nested etc but no dice, currently it seems to not pass any requests to /api to the proxy

upstream node_upstream {
    server 127.0.0.1:3000 fail_timeout=0;
}

server {
     listen 80;
     listen [::]:80;
     server_name example.com;

     location /api {
         allow all;
     }

     location / {
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_redirect off;
         proxy_buffering off;
         proxy_pass http://node_upstream;
         allow 1.2.3.4;
         deny all;
     }

     location /public {
         root /path/to/static/files;
     }

     listen 443 ssl;
     ... SSL Config here...

     if ($scheme != "https") {
         return 301 https://$host$request_uri;
     }
}