Nginx: protect directory and files

I wrote this to prevent the directory "download" to be accessed but from the IP 1.2.3.4.

location ~ /folder/download {
    allow 1.2.3.4;
    deny all;
    return 403;
}

However, the directory "folder" is being blocked as well and I did not wanted this.

What am I doing wrong?

UPDATE:

Here it goes all the real config:

server {
    server_name www.domain.com;
    rewrite ^ $scheme://domain.com$request_uri? permanent;
}

server {
    server_name atpc.info;

    access_log /var/log/nginx/atpc.info.access;
    error_log /var/log/nginx/atpc.info.error;

    root /var/www/atpc.info;

    location ^~ folder/download {
            allow 1.2.3.4;
            deny all;
    }

    location ^~ folder/includes {
            allow 1.2.3.4;
            deny all;
    }

    location ^~ folder/mythings {
            allow 1.2.3.4;
            deny all;
    }

    location ^~ folder/functions {
            allow 1.2.3.4;
            deny all;
    }

    location / { index index.htm index.php; }

    location ~ .php$ {
            fastcgi_pass 127.0.0.1:9000;
            fastcgi_index index.php;
            fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
            include /etc/nginx/fastcgi_params;
    }

    location = /favicon.ico {
            return 204;
            access_log off;
            log_not_found off;
    }

    location = /robots.txt { allow all; log_not_found off; access_log off; }
    location ~ /\. { deny all; access_log off; log_not_found off; }
}

Thanks.