Flexible management of map,geo and rate_limit instances
This patch gives full flexibility in managing mentioned instances
in nginx configurations.
Co-Authored-By: Oleksandr Bryndzii <obryndzii@mirantis.com>
Related-Prod: PROD-24400
Change-Id: Iebdc66351c24c9c847d6d4d0c6d4efcf1b1a57ae
diff --git a/README.rst b/README.rst
index eadade3..0e04cf8 100644
--- a/README.rst
+++ b/README.rst
@@ -271,41 +271,149 @@
name: gitlab.domain.com
port: 80
-Proxy with rate limiting scheme:
+Use nginx `ngx_http_map_module` that creates variables whose values depend on
+values of other variables.
.. code-block:: yaml
- _dollar: '$'
+ nginx:
+ server:
+ enabled: true
+ map:
+ enabled: true
+ items:
+ mymap:
+ enabled: true
+ string: input_string
+ variable: output_map_variable
+ body:
+ default:
+ value: '""'
+ example.com:
+ value: '1'
+ example.org:
+ value: '2'
+
+Use nginx `ngx_http_geo_module module` that creates variables with values
+depending on the client IP address.
+
+.. code-block:: yaml
+
+ nginx:
+ server:
+ enabled: true
+ geo:
+ enabled: true
+ items:
+ my_geo_map:
+ enabled: true
+ variable: output_get_variable
+ body:
+ default:
+ value: '""'
+ cl1
+ name: 10.12.100.1/32
+ value: '1'
+ cl2
+ name: 10.13.0.0/16
+ value: 2'
+
+Use `ngx_http_limit_req_module` module that is used to limit the request
+processing rate per a defined key, in particular, the processing rate of
+requests coming from a single IP address. The limitation is done using
+the `leaky bucket` method.
+The `limit_req_module` might be configured globally or applied to specific
+nginx site.
+
+.. code-block:: yaml
+
+ nginx:
+ server:
+ limit_req_module:
+ limit_req_zone:
+ global_limit_ip_zone:
+ key: global_limit_ip_var
+ size: 10m
+ rate: '1r/s'
+ limit_req_status: 503
+ limit_req:
+ global_limit_zone:
+ burst: 5
+ enabled: true
+
+There is an example to to limit requests to all sites based on IP.
+In the following example all clients are limited except of 10.12.100.1
+with 1 req per second.
+
+#. Create geo instance that will match IP and set `limit_action` var.
+ "0" - is unlimited, 1 - limited
+
+#. Create a `global_geo_limiting_map` that will map `ip_limit_key` to
+ `ip_limit_action`
+
+#. Create global `limit_req_zone` called `global_limit_zone` that limits
+ number of requests to 1r/s
+
+#. Apply `global_limit_zone` globally to all requests with 5 req burst.
+
+.. code-block:: yaml
+
+ nginx:
+ server:
+ enabled: true
+ geo:
+ enabled: true
+ items:
+ global_geo_limiting:
+ enabled: true
+ variable: ip_limit_key
+ body:
+ default:
+ value: '1'
+ unlimited_client1:
+ name: '10.12.100.1/32'
+ value: '0'
+ map:
+ enabled: true
+ items:
+ global_geo_limiting_map:
+ enabled: true
+ string: ip_limit_key
+ variable: ip_limit_action
+ body:
+ limited:
+ name: 1
+ value: '$binary_remote_addr'
+ unlimited:
+ name: 0
+ value: '""'
+ limit_req_module:
+ limit_req_zone:
+ global_limit_zone:
+ key: ip_limit_action
+ size: 10m
+ rate: '1r/s'
+ limit_req_status: 503
+ limit_req:
+ global_limit_zone:
+ burst: 5
+ enabled: true
+
+To apply request limiting to particular site only `limit_req` should be
+applied on site level, for example:
+
+.. code-block:: yaml
+
nginx:
server:
site:
- nginx_proxy_site01:
- enabled: true
- type: nginx_proxy
- name: site01
- proxy:
- host: local.domain.com
- port: 80
- protocol: http
- host:
- name: gitlab.domain.com
- port: 80
- limit:
- enabled: True
- ip_whitelist:
- - 127.0.0.1
- burst: 600
- rate: 10r/s
- nodelay: True
- subfilters:
- heavy_url:
- input: ${_dollar}{binary_remote_addr}${_dollar}{request_uri}
- mode: blacklist
- items:
- - "~.*servers/detail[?]name=.*&status=ACTIVE"
- rate: 2r/m
- burst: 2
- nodelay: True
+ nginx_proxy_openstack_api_keystone:
+ limit_req_module:
+ limit_req:
+ global_limit_zone:
+ burst: 5
+ enabled: true
+
Gitlab server with user for basic auth:
diff --git a/nginx/files/_geo.conf b/nginx/files/_geo.conf
new file mode 100644
index 0000000..6c127b5
--- /dev/null
+++ b/nginx/files/_geo.conf
@@ -0,0 +1,9 @@
+{%- for gname,gdata in _data.get('items', {}).iteritems() %}
+geo {{ '$' + _gdata.address if gdata.address is defined else '' }} ${{gdata.variable}} {
+{%- for _item,_value in gdata.body.iteritems() %}
+ {%- if _value.get('enabled', 'True') %}
+ {{ _value.get('name', _item) }} {{ _value.value }};
+ {%- endif %}
+{%- endfor %}
+}
+{%- endfor %}
diff --git a/nginx/files/_limit_req_module.conf b/nginx/files/_limit_req_module.conf
new file mode 100644
index 0000000..26971ea
--- /dev/null
+++ b/nginx/files/_limit_req_module.conf
@@ -0,0 +1,19 @@
+{%- for _item,_value in _data.get('limit_req_zone', {}).iteritems() %}
+ {%- if _value.get('enabled', 'True') %}
+limit_req_zone ${{ _value.key }} zone={{ _value.get('name', _item) }}:{{ _value.size}} rate={{ _value.rate }} {% if _value.get('sync') %}{{ sync }}{% endif %};
+ {%- endif %}
+{%- endfor %}
+
+{%- for _item,_value in _data.get('limit_req', {}).iteritems() %}
+ {%- if _value.get('enabled', 'True') %}
+limit_req zone={{ _value.get('name', _item) }} {% if _value.get('burst') %}burst={{ _value.burst }}{% endif %} {% if _value.get('nodelay') %}nodelay{% endif %} {% if _value.delay is defined %}delay={{ _value.delay }}{% endif %};
+ {%- endif %}
+{%- endfor %}
+
+{%- if _data.limit_req_status is defined %}
+limit_req_status {{ _data.limit_req_status }};
+{%- endif %}
+
+{%- if _data.limit_req_log_level is defined %}
+limit_req_log_level {{ _data.limit_req_log_level }};
+{% endif %}
diff --git a/nginx/files/_map.conf b/nginx/files/_map.conf
new file mode 100644
index 0000000..086de4e
--- /dev/null
+++ b/nginx/files/_map.conf
@@ -0,0 +1,9 @@
+{%- for mname,mdata in _data.get('items', {}).iteritems() %}
+map {{ '$' + mdata.string if mdata.string is defined else '' }} ${{mdata.variable}} {
+{%- for _item,_value in mdata.body.iteritems() %}
+ {%- if _value.get('enabled', 'True') %}
+ {{ _value.get('name', _item) }} {{ _value.value }};
+ {%- endif %}
+{%- endfor %}
+}
+{%- endfor %}
diff --git a/nginx/files/nginx.conf b/nginx/files/nginx.conf
index c70f7a0..5a15b4a 100644
--- a/nginx/files/nginx.conf
+++ b/nginx/files/nginx.conf
@@ -78,8 +78,15 @@
# Virtual Host Configs
##
+
+{%- if server.limit_req_module is defined %}
+ {%- set _data = server.limit_req_module %}
+ {%- include "nginx/files/_limit_req_module.conf" %}
+{%- endif %}
+
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*.conf;
+
}
{% if server.stream is defined %}
diff --git a/nginx/files/proxy.conf b/nginx/files/proxy.conf
index e2036f0..2353c93 100644
--- a/nginx/files/proxy.conf
+++ b/nginx/files/proxy.conf
@@ -128,6 +128,13 @@
{%- endif %}
{%- endif %}
+ {%- if site.limit_req_module is defined %}
+ {%- set _data = site.limit_req_module %}
+ {%- include "nginx/files/_limit_req_module.conf" %}
+ {%- endif %}
+
+ {# The approach below is deprecated, as it was limited funtionality #}
+ {# compare to flexibility that nginx provide. site:limit_req_module:limit_req shall be used instead. #}
{%- if site.get('limit', {}).get('enabled', False) %}
limit_req zone={{ site_name }}{% if site.limit.get('burst', False) %} burst={{ site.limit.burst }}{% endif %}{% if site.limit.get('nodelay', False) %} nodelay{% endif %};
{%- for subfilter_name, subfilter in site.limit.get('subfilters', {}).items() %}
diff --git a/nginx/server.sls b/nginx/server.sls
index b7896d3..45339d0 100644
--- a/nginx/server.sls
+++ b/nginx/server.sls
@@ -79,6 +79,31 @@
- pkg: nginx_packages
{%- endif %}
+{%- if server.get('geo', {}).get('enabled', False) %}
+/etc/nginx/conf.d/geo.conf:
+ file.managed:
+ - source: salt://nginx/files/_geo.conf
+ - template: jinja
+ - require:
+ - pkg: nginx_packages
+ - watch_in:
+ - service: nginx_service
+ - defaults:
+ _data: {{ server.geo }}
+{%- endif %}
+{%- if server.get('map', {}).get('enabled', False) %}
+/etc/nginx/conf.d/map.conf:
+ file.managed:
+ - source: salt://nginx/files/_map.conf
+ - template: jinja
+ - require:
+ - pkg: nginx_packages
+ - watch_in:
+ - service: nginx_service
+ - defaults:
+ _data: {{ server.map }}
+{%- endif %}
+
{%- if server.stream is defined %}
/etc/nginx/stream.conf:
file.managed:
diff --git a/tests/pillar/proxy_rate_limit.sls b/tests/pillar/proxy_rate_limit.sls
index 4e41fa9..7dc7156 100644
--- a/tests/pillar/proxy_rate_limit.sls
+++ b/tests/pillar/proxy_rate_limit.sls
@@ -9,6 +9,39 @@
bind:
address: 127.0.0.1
protocol: tcp
+ geo:
+ enabled: true
+ items:
+ global_geo_limiting:
+ enabled: true
+ variable: ip_limit_key
+ body:
+ default:
+ value: '1'
+ unlimited_client1:
+ name: '10.12.100.1/32'
+ value: '0'
+ map:
+ enabled: true
+ items:
+ global_geo_limiting_map:
+ enabled: true
+ string: ip_limit_key
+ variable: ip_limit_action
+ body:
+ limited:
+ name: 1
+ value: '$binary_remote_addr'
+ unlimited:
+ name: 0
+ value: '""'
+ limit_req_module:
+ limit_req_zone:
+ global_limit_zone:
+ key: ip_limit_action
+ size: 10m
+ rate: '1r/s'
+ limit_req_status: 503
site:
nginx_proxy_site01:
enabled: true
@@ -21,27 +54,8 @@
host:
name: cloudlab.domain.com
port: 31337
- limit:
- enabled: True
- ip_whitelist:
- - 127.0.0.1
- burst: 600
- rate: 10r/s
- nodelay: True
- subfilters:
- show_active_instance:
- input: ${_dollar}{binary_remote_addr}${_dollar}{request_uri}
- mode: blacklist
- items:
- - "~.*servers/detail[?]name=.*&status=ACTIVE"
- rate: 2r/m
- burst: 2
- nodelay: True
- server_list:
- input: ${_dollar}{binary_remote_addr}${_dollar}{request_uri}
- mode: blacklist
- items:
- - "~.*servers/detail$"
- rate: 30r/m
- burst: 20
- nodelay: True
+ limit_req_module:
+ limit_req:
+ global_limit_zone:
+ burst: 5
+ enabled: true