Allow defining robots.txt
diff --git a/apache/files/_locations.conf b/apache/files/_locations.conf
index d9bb74d..0366cf8 100644
--- a/apache/files/_locations.conf
+++ b/apache/files/_locations.conf
@@ -1,3 +1,10 @@
+{%- from "apache/map.jinja" import server with context -%}
+ {%- if site.robots is defined %}
+ <Location "/robots.txt">
+ SetHandler None
+ </Location>
+ Alias /robots.txt {{ server.www_dir }}/{{ site_robots }}.txt
+ {%- endif %}
{%- if site.locations is defined %}
{%- for location in site.locations %}
diff --git a/apache/files/heka.toml b/apache/files/heka.toml
index bff6ff7..6d0bbe6 100644
--- a/apache/files/heka.toml
+++ b/apache/files/heka.toml
@@ -16,20 +16,20 @@
differentiator = ["apache-", "SiteName", "-access"]
decoder = "Sandbox_apache_access_vhost"
-[logstreamer_apache_error]
-type = "LogstreamerInput"
-log_directory = "{{ server.log_dir }}"
-file_match = 'error\.log\.?(?P<Index>\d+)?(\.gz)?'
-priority = ["^Index"]
-decoder = "Sandbox_apache_error"
-
-[logstreamer_apache_site_error]
-type = "LogstreamerInput"
-log_directory = "{{ server.log_dir }}"
-file_match = '(?P<SiteName>.+)[\.\-]error\.log\.?(?P<Index>\d*)?(\.gz)?'
-priority = ["^Index"]
-differentiator = ["apache-", "SiteName", "-error"]
-decoder = "Sandbox_apache_error"
+#[logstreamer_apache_error]
+#type = "LogstreamerInput"
+#log_directory = "{{ server.log_dir }}"
+#file_match = 'error\.log\.?(?P<Index>\d+)?(\.gz)?'
+#priority = ["^Index"]
+#decoder = "Sandbox_apache_error"
+#
+#[logstreamer_apache_site_error]
+#type = "LogstreamerInput"
+#log_directory = "{{ server.log_dir }}"
+#file_match = '(?P<SiteName>.+)[\.\-]error\.log\.?(?P<Index>\d*)?(\.gz)?'
+#priority = ["^Index"]
+#differentiator = ["apache-", "SiteName", "-error"]
+#decoder = "Sandbox_apache_error"
## Sandboxes
@@ -51,9 +51,9 @@
user_agent_transform = true
log_format = '%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"'
-[Sandbox_apache_error]
-type = "SandboxDecoder"
-filename = "lua_decoders/decoders/apache_error.lua"
-
-[Sandbox_apache_error.config]
-type = "apache_error"
+#[Sandbox_apache_error]
+#type = "SandboxDecoder"
+#filename = "lua_decoders/decoders/apache_error.lua"
+#
+#[Sandbox_apache_error.config]
+#type = "apache_error"
diff --git a/apache/files/robots.txt b/apache/files/robots.txt
new file mode 100644
index 0000000..26c434a
--- /dev/null
+++ b/apache/files/robots.txt
@@ -0,0 +1,13 @@
+{%- set robots = salt['pillar.get']('apache:server:robots:'+robots_name) -%}
+{%- for robot, opts in robots.iteritems() -%}
+User-agent: {% if robot == 'all' %}*{% else %}{{ robot }}{% endif %}
+{%- if opts.disallow %}
+{%- for path in opts.disallow %}
+Disallow: {{ path }}
+{%- endfor %}
+{%- else %}
+Disallow:
+{%- endif %}
+{%- endfor %}
+
+{#- vim: syntax=jinja -#}
diff --git a/apache/map.jinja b/apache/map.jinja
index dde5d4f..8e9566c 100644
--- a/apache/map.jinja
+++ b/apache/map.jinja
@@ -13,7 +13,7 @@
'conf_dir': '/etc/apache2/conf-available',
'conf_ext': '.conf',
'log_dir': '/var/log/apache2',
- 'wwwdir': '/srv',
+ 'www_dir': '/var/www',
'service_user': 'www-data',
'service_group': 'www-data',
},
@@ -27,7 +27,7 @@
'conf_dir': '/etc/httpd/conf',
'conf_ext': '.conf',
'log_dir': '/var/log/httpd',
- 'wwwdir': '/srv',
+ 'www_dir': '/var/www',
'service_user': 'apache',
'service_group': 'apache',
},
diff --git a/apache/server/init.sls b/apache/server/init.sls
index 46d4d4f..797345b 100644
--- a/apache/server/init.sls
+++ b/apache/server/init.sls
@@ -2,3 +2,4 @@
- apache.server.service
- apache.server.site
- apache.server.users
+- apache.server.robots
diff --git a/apache/server/robots.sls b/apache/server/robots.sls
new file mode 100644
index 0000000..9001343
--- /dev/null
+++ b/apache/server/robots.sls
@@ -0,0 +1,14 @@
+{%- from "apache/map.jinja" import server with context %}
+{%- if server.enabled %}
+
+{%- for name, robots in server.get('robots', {}).iteritems() %}
+robots_{{ name }}:
+ file.managed:
+ - name: {{ server.www_dir }}/robots_{{ name }}.txt
+ - template: jinja
+ - source: salt://apache/files/robots.txt
+ - defaults:
+ robots_name: "{{ name }}"
+{%- endfor %}
+
+{%- endif %}
diff --git a/tests/pillar/apache_server.sls b/tests/pillar/apache_server.sls
index da59713..fe53ffa 100644
--- a/tests/pillar/apache_server.sls
+++ b/tests/pillar/apache_server.sls
@@ -9,12 +9,18 @@
enabled: true
password: cloudlab
htpasswd: cloudlab.htpasswd
+ robots:
+ default:
+ all:
+ disallow:
+ - /
site:
roundcube:
enabled: true
type: static
name: roundcube
root: /usr/share/roundcube
+ robots: default
locations:
- uri: /admin
path: /usr/share/postfixadmin