Merge pull request #34 from mnederlof/master
Implement rate limiting for nginx proxies
diff --git a/.kitchen.yml b/.kitchen.yml
index 2db27ce..cfb7f2a 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -60,6 +60,11 @@
pillars-from-files:
nginx.sls: tests/pillar/proxy.sls
+ - name: proxy-rate-limit
+ provisioner:
+ pillars-from-files:
+ nginx.sls: tests/pillar/proxy_rate_limit.sls
+
- name: redirect
provisioner:
pillars-from-files:
diff --git a/.travis.yml b/.travis.yml
index 9071f65..fee457f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -20,12 +20,14 @@
- PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2016.3 SUITE=horizon-no-ssl
- PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2016.3 SUITE=horizon-with-ssl
- PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2016.3 SUITE=proxy
+ - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2016.3 SUITE=proxy-rate-limit
- PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2016.3 SUITE=redirect
- PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2016.3 SUITE=static
- PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2016.3 SUITE=stats
- PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2017.7 SUITE=horizon-no-ssl
- PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2017.7 SUITE=horizon-with-ssl
- PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2017.7 SUITE=proxy
+ - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2017.7 SUITE=proxy-rate-limit
- PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2017.7 SUITE=redirect
- PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2017.7 SUITE=static
- PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2017.7 SUITE=stats
diff --git a/README.rst b/README.rst
index df3ba3b..4eadf2b 100644
--- a/README.rst
+++ b/README.rst
@@ -264,6 +264,42 @@
name: gitlab.domain.com
port: 80
+Proxy with rate limiting scheme:
+
+.. code-block:: yaml
+
+ _dollar: '$'
+ nginx:
+ server:
+ site:
+ nginx_proxy_site01:
+ enabled: true
+ type: nginx_proxy
+ name: site01
+ proxy:
+ host: local.domain.com
+ port: 80
+ protocol: http
+ host:
+ name: gitlab.domain.com
+ port: 80
+ limit:
+ enabled: True
+ ip_whitelist:
+ - 127.0.0.1
+ burst: 600
+ rate: 10r/s
+ nodelay: True
+ subfilters:
+ heavy_url:
+ input: ${_dollar}{binary_remote_addr}${_dollar}{request_uri}
+ mode: blacklist
+ items:
+ - "~.*servers/detail[?]name=.*&status=ACTIVE"
+ rate: 2r/m
+ burst: 2
+ nodelay: True
+
Gitlab server with user for basic auth
.. code-block:: yaml
diff --git a/nginx/files/_limit.conf b/nginx/files/_limit.conf
new file mode 100644
index 0000000..e0ff102
--- /dev/null
+++ b/nginx/files/_limit.conf
@@ -0,0 +1,31 @@
+{%- set site = salt['pillar.get']('nginx:server:site:'+site_name) %}
+
+{%- if site.get('limit', {}).get('enabled', False) %}
+# Create whitelist for ip addresses
+geo $geo_{{ site_name }} {
+ default "enforce";
+{%- for ip in site.limit.get('ip_whitelist', []) %}
+ {{ ip }} "whitelist";
+{%- endfor %}
+}
+
+# First, map all whitelisted IP's to the request query
+map $geo_{{ site_name }} $limit_{{ site_name }} {
+ default {{ site.limit.get('query', '$binary_remote_addr') }};
+ "whitelist" "";
+}
+limit_req_zone $limit_{{ site_name }} zone={{ site_name }}:{{ site.limit.get('size', '100m') }} rate={{ site.limit.get('rate', '30r/m') }};
+
+{%- for subfilter_name, subfilter in site.limit.get('subfilters', {}).items() %}
+
+map "${geo_{{ site_name }}}{{ subfilter.get('input', '$limit_{{ site_name }}') }}" $limit_{{ site_name }}_{{ subfilter_name }} {
+ default {% if subfilter.get('mode', 'whitelist') == "whitelist" %}"{{ subfilter.get('input', '$limit_{{ site_name }}') }}";{% else %}""{% endif %};
+ "~^whitelist" ""; # Allow previously whitelisted results.
+{%- for match in subfilter.get('items', []) %}
+ "{{ match }}" {% if subfilter.get('mode', 'whitelist') == 'whitelist' %}""{% else %}"{{ subfilter.get('input', '$limit_{{ site_name }}') }}"{% endif %};
+{%- endfor %}
+}
+limit_req_zone $limit_{{ site_name }}_{{ subfilter_name }} zone={{ site_name }}_{{ subfilter_name }}:{{ subfilter.get('size', site.limit.get('size', '100m')) }} rate={{ subfilter.get('rate', site.limit.get('rate', '30r/m')) }};
+{%- endfor %}
+
+{%- endif %}
diff --git a/nginx/files/nginx.conf b/nginx/files/nginx.conf
index 703fc2e..9f19e7d 100644
--- a/nginx/files/nginx.conf
+++ b/nginx/files/nginx.conf
@@ -26,6 +26,8 @@
server_names_hash_bucket_size 128;
# server_name_in_redirect off;
+ variables_hash_bucket_size {{server.get('variables_hash_bucket_size', '128') }};
+
include /etc/nginx/mime.types;
default_type application/octet-stream;
diff --git a/nginx/files/proxy.conf b/nginx/files/proxy.conf
index bcbb6bd..8b4601a 100644
--- a/nginx/files/proxy.conf
+++ b/nginx/files/proxy.conf
@@ -1,5 +1,7 @@
{%- set site = salt['pillar.get']('nginx:server:site:'+site_name) %}
+{%- include "nginx/files/_limit.conf" %}
+
server {
{%- include "nginx/files/_name.conf" %}
@@ -22,7 +24,10 @@
{# If location is not defined in model, use proxy definition by default #}
{%- do location.update({'/': site.proxy}) %}
{%- endif %}
-
+ {%- if site.get('limit', {}).get('enabled', False) %}
+ limit_req_status {{ site.limit.get('status_code', '429') }};
+ limit_conn_status {{ site.limit.get('status_code', '429') }};
+ {%- endif %}
{%- for path, location in location.items() %}
location {{ path }} {
{%- if location.upstream_proxy_pass is defined %}
@@ -104,6 +109,12 @@
{%- endif %}
{%- endif %}
+ {%- if site.get('limit', {}).get('enabled', False) %}
+ limit_req zone={{ site_name }}{% if site.limit.get('burst', False) %} burst={{ site.limit.burst }}{% endif %}{% if site.limit.get('nodelay', False) %} nodelay{% endif %};
+ {%- for subfilter_name, subfilter in site.limit.get('subfilters', {}).items() %}
+ limit_req zone={{ site_name }}_{{ subfilter_name }}{% if subfilter.get('burst', False) %} burst={{ subfilter.burst }}{% endif %}{% if subfilter.get('nodelay', False) %} nodelay{% endif %};
+ {%- endfor %}
+ {%- endif %}
}
{%- endfor %}
}
diff --git a/tests/pillar/proxy_rate_limit.sls b/tests/pillar/proxy_rate_limit.sls
new file mode 100644
index 0000000..4e41fa9
--- /dev/null
+++ b/tests/pillar/proxy_rate_limit.sls
@@ -0,0 +1,47 @@
+_dollar: '$'
+salt:
+ minion:
+ enabled: true
+nginx:
+ server:
+ enabled: true
+ extras: false
+ bind:
+ address: 127.0.0.1
+ protocol: tcp
+ site:
+ nginx_proxy_site01:
+ enabled: true
+ type: nginx_proxy
+ name: site01
+ proxy:
+ host: 127.0.0.1
+ port: 808
+ protocol: http
+ host:
+ name: cloudlab.domain.com
+ port: 31337
+ limit:
+ enabled: True
+ ip_whitelist:
+ - 127.0.0.1
+ burst: 600
+ rate: 10r/s
+ nodelay: True
+ subfilters:
+ show_active_instance:
+ input: ${_dollar}{binary_remote_addr}${_dollar}{request_uri}
+ mode: blacklist
+ items:
+ - "~.*servers/detail[?]name=.*&status=ACTIVE"
+ rate: 2r/m
+ burst: 2
+ nodelay: True
+ server_list:
+ input: ${_dollar}{binary_remote_addr}${_dollar}{request_uri}
+ mode: blacklist
+ items:
+ - "~.*servers/detail$"
+ rate: 30r/m
+ burst: 20
+ nodelay: True