Merge pull request #72 from aplsms/feature-tap_custom_txqueuelen
Feature: automatically set txqueuelen for all tap* network interfaces
diff --git a/README.rst b/README.rst
index 7860bdf..c16d319 100644
--- a/README.rst
+++ b/README.rst
@@ -445,6 +445,69 @@
priority: 900
package: '*'
+
+Package manager proxy setup globally:
+
+.. code-block:: yaml
+
+ linux:
+ system:
+ ...
+ repo:
+ apt-mk:
+ source: "deb http://apt-mk.mirantis.com/ stable main salt"
+ ...
+ proxy:
+ pkg:
+ enabled: true
+ ftp: ftp://ftp-proxy-for-apt.host.local:2121
+ ...
+ # NOTE: Global defaults for any other componet that configure proxy on the system.
+ # If your environment has just one simple proxy, set it on linux:system:proxy.
+ #
+ # fall back system defaults if linux:system:proxy:pkg has no protocol specific entries
+ # as for https and http
+ ftp: ftp://proxy.host.local:2121
+ http: http://proxy.host.local:3142
+ https: https://proxy.host.local:3143
+
+Package manager proxy setup per repository:
+
+.. code-block:: yaml
+
+ linux:
+ system:
+ ...
+ repo:
+ debian:
+ source: "deb http://apt-mk.mirantis.com/ stable main salt"
+ ...
+ apt-mk:
+ source: "deb http://apt-mk.mirantis.com/ stable main salt"
+ # per repository proxy
+ proxy:
+ enabled: true
+ http: http://maas-01:8080
+ https: http://maas-01:8080
+ ...
+ proxy:
+ # package manager fallback defaults
+ # used if linux:system:repo:apt-mk:proxy has no protocol specific entries
+ pkg:
+ enabled: true
+ ftp: ftp://proxy.host.local:2121
+ #http: http://proxy.host.local:3142
+ #https: https://proxy.host.local:3143
+ ...
+ # global system fallback system defaults
+ ftp: ftp://proxy.host.local:2121
+ http: http://proxy.host.local:3142
+ https: https://proxy.host.local:3143
+
+
+RC
+~~
+
rc.local example
.. code-block:: yaml
@@ -467,6 +530,7 @@
# By default this script does nothing.
exit 0
+
Prompt
~~~~~~
@@ -708,18 +772,70 @@
use_interfaces:
- eth1
-Linux with proxy
+Configure global environment variables
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Linux /etc/environment:
+``/etc/environment`` is for static system wide variable assignment after boot. Variable expansion is frequently not supported.
.. code-block:: yaml
linux:
- network:
+ system:
+ env:
+ BOB_VARIABLE: Alice
+ ...
+ BOB_PATH:
+ - /srv/alice/bin
+ - /srv/bob/bin
+ ...
+ ftp_proxy: none
+ http_proxy: http://global-http-proxy.host.local:8080
+ https_proxy: ${linux:system:proxy:https}
+ no_proxy:
+ - 192.168.0.80
+ - 192.168.1.80
+ - .domain.com
+ - .local
...
+ # NOTE: global defaults proxy configuration.
proxy:
- host: proxy.domain.com
- port: 3128
+ ftp: ftp://proxy.host.local:2121
+ http: http://proxy.host.local:3142
+ https: https://proxy.host.local:3143
+ noproxy:
+ - .domain.com
+ - .local
+
+Configure profile.d scripts
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Linux /etc/profile.d:
+The profile.d scripts are being sourced during .sh execution and support variable expansion in opposite to /etc/environment
+global settings in ``/etc/environment``.
+
+.. code-block:: yaml
+
+ linux:
+ system:
+ profile:
+ locales: |
+ export LANG=C
+ export LC_ALL=C
+ ...
+ vi_flavors.sh: |
+ export PAGER=view
+ export EDITOR=vim
+ alias vi=vim
+ shell_locales.sh: |
+ export LANG=en_US
+ export LC_ALL=en_US.UTF-8
+ shell_proxies.sh: |
+ export FTP_PROXY=ftp://127.0.3.3:2121
+ export NO_PROXY='.local'
Linux with hosts
+~~~~~~~~~~~~~~~~
Parameter purge_hosts will enforce whole /etc/hosts file, removing entries
that are not defined in model except defaults for both IPv4 and IPv6 localhost
@@ -753,6 +869,7 @@
Setup resolv.conf, nameservers, domain and search domains
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. code-block:: yaml
diff --git a/linux/files/apt.conf.d_proxies b/linux/files/apt.conf.d_proxies
new file mode 100644
index 0000000..ca53711
--- /dev/null
+++ b/linux/files/apt.conf.d_proxies
@@ -0,0 +1,9 @@
+{%- if ftp and ftp.lower() != 'none' %}
+Acquire::ftp::proxy{%- if external_host %}::{{ external_host }}{% endif %} "{{ ftp }}";
+{%- endif %}
+{%- if http and http.lower() != 'none' %}
+Acquire::http::proxy{%- if external_host %}::{{ external_host }}{% endif %} "{{ http }}";
+{%- endif %}
+{%- if https and https.lower() != 'none' %}
+Acquire::https::proxy{%- if external_host %}::{{ external_host }}{% endif %} "{{ https }}";
+{%- endif -%}
diff --git a/linux/files/etc_environment b/linux/files/etc_environment
new file mode 100644
index 0000000..3b6f480
--- /dev/null
+++ b/linux/files/etc_environment
@@ -0,0 +1,32 @@
+
+{%- for name,value in variables.iteritems() if not name.lower().endswith('_proxy') %}
+
+{%- if value is sequence and value is not string %}
+{{ name }}="{{ value|join(':') }}"
+
+{%- else %}
+{{ name }}="{{ value }}"
+
+{%- endif %}
+{%- endfor %}
+
+{%- if ftp_proxy and ftp_proxy.lower() != 'none' %}
+ftp_proxy="{{ ftp_proxy }}";
+FTP_PROXY="{{ ftp_proxy }}";
+{%- endif %}
+
+{%- if http_proxy and http_proxy.lower() != 'none' %}
+http_proxy="{{ http_proxy }}";
+HTTP_PROXY="{{ http_proxy }}";
+{%- endif %}
+
+{%- if https_proxy and https_proxy.lower() != 'none' %}
+https_proxy="{{ https_proxy }}";
+HTTPS_PROXY="{{ https_proxy }}";
+{%- endif %}
+
+{%- if no_proxy %}
+no_proxy="{{ no_proxy|join(',') }}";
+NO_PROXY="{{ no_proxy|join(',') }}";
+{%- endif %}
+
diff --git a/linux/files/etc_profile b/linux/files/etc_profile
new file mode 100644
index 0000000..73bb629
--- /dev/null
+++ b/linux/files/etc_profile
@@ -0,0 +1 @@
+{{ script }}
diff --git a/linux/files/etc_profile_vi_flavors.sh b/linux/files/etc_profile_vi_flavors.sh
new file mode 100644
index 0000000..e74c214
--- /dev/null
+++ b/linux/files/etc_profile_vi_flavors.sh
@@ -0,0 +1,3 @@
+set -o vi
+export EDITOR=vim
+{{ script }}
diff --git a/linux/map.jinja b/linux/map.jinja
index b49ff66..25eefae 100644
--- a/linux/map.jinja
+++ b/linux/map.jinja
@@ -8,6 +8,9 @@
'limit': {},
'locale': {},
'motd': {},
+ 'env': {},
+ 'profile': {},
+ 'proxy': {},
'repo': {},
'package': {},
'autoupdates': {
@@ -26,6 +29,9 @@
'limit': {},
'locale': {},
'motd': {},
+ 'env': {},
+ 'profile': {},
+ 'proxy': {},
'repo': {},
'package': {},
'autoupdates': {
@@ -44,6 +50,9 @@
'limit': {},
'locale': {},
'motd': {},
+ 'env': {},
+ 'profile': {},
+ 'proxy': {},
'repo': {},
'package': {},
'autoupdates': {
@@ -112,7 +121,7 @@
'interface_params': interface_params,
'bridge': 'none',
'proxy': {
- 'host': 'none',
+ 'host': 'none'
},
'host': {},
},
@@ -125,7 +134,7 @@
'interface_params': interface_params,
'bridge': 'none',
'proxy': {
- 'host': 'none',
+ 'host': 'none'
},
'host': {},
},
diff --git a/linux/meta/heka.yml b/linux/meta/heka.yml
index d3abda2..d45504d 100644
--- a/linux/meta/heka.yml
+++ b/linux/meta/heka.yml
@@ -17,7 +17,7 @@
function: avg
linux_system_cpu_warning:
description: 'The CPU wait times are high.'
- severity: critical
+ severity: warning
rules:
- metric: cpu_wait
relational_operator: '>='
diff --git a/linux/network/interface.sls b/linux/network/interface.sls
index 089ab9e..bed9a3f 100644
--- a/linux/network/interface.sls
+++ b/linux/network/interface.sls
@@ -77,11 +77,15 @@
linux_interfaces_include_{{ interface_name }}:
file.prepend:
- name: /etc/network/interfaces
- - text: 'source /etc/network/interfaces.d/*'
+ - text: |
+ source /etc/network/interfaces.d/*
+ # Workaround for Upstream-Bug: https://github.com/saltstack/salt/issues/40262
+ source /etc/network/interfaces.u/*
ovs_port_{{ interface_name }}:
file.managed:
- - name: /etc/network/interfaces.d/ifcfg-{{ interface_name }}
+ - name: /etc/network/interfaces.u/ifcfg-{{ interface_name }}
+ - makedirs: True
- source: salt://linux/files/ovs_port
- defaults:
port: {{ interface|yaml }}
@@ -272,7 +276,10 @@
linux_interfaces_final_include:
file.prepend:
- name: /etc/network/interfaces
- - text: 'source /etc/network/interfaces.d/*'
+ - text: |
+ source /etc/network/interfaces.d/*
+ # Workaround for Upstream-Bug: https://github.com/saltstack/salt/issues/40262
+ source /etc/network/interfaces.u/*
{%- endif %}
diff --git a/linux/system/env.sls b/linux/system/env.sls
new file mode 100644
index 0000000..8b3aade
--- /dev/null
+++ b/linux/system/env.sls
@@ -0,0 +1,36 @@
+{%- from "linux/map.jinja" import system with context %}
+{%- if system.enabled %}
+
+{%- if system.env|length > 0 %}
+
+linux_system_environment_proxies:
+ file.blockreplace:
+ - name: /etc/environment
+ - marker_start: '# START - SALT MANAGED VARIABLES, DO NOT EDIT'
+ - marker_end: '# END - SALT MANAGED VARIABLES'
+ - template: jinja
+ - source: salt://linux/files/etc_environment
+ - append_if_not_found: True
+ - backup: '.bak'
+ - show_changes: True
+ - defaults:
+ variables: {{ system.env | yaml }}
+ no_proxy: {{ system.env.get('no_proxy', None) }}
+ https_proxy: {{ system.env.get('https_proxy', None) }}
+ http_proxy: {{ system.env.get('http_proxy', None) }}
+ ftp_proxy: {{ system.env.get('ftp_proxy', None) }}
+
+{%- else %}
+
+linux_system_environment_proxies:
+ file.blockreplace:
+ - name: /etc/environment
+ - marker_start: '# SALT MANAGED VARIABLES - DO NOT EDIT - START'
+ - content: '# '
+ - marker_end: '# SALT MANAGED VARIABLES - END'
+ - append_if_not_found: True
+ - backup: '.bak'
+ - show_changes: True
+
+{%- endif %}
+{%- endif %}
diff --git a/linux/system/init.sls b/linux/system/init.sls
index 9d4d4f0..84c2365 100644
--- a/linux/system/init.sls
+++ b/linux/system/init.sls
@@ -1,5 +1,7 @@
{%- from "linux/map.jinja" import system with context %}
include:
+- linux.system.env
+- linux.system.profile
{%- if system.repo|length > 0 %}
- linux.system.repo
{%- endif %}
diff --git a/linux/system/profile.sls b/linux/system/profile.sls
new file mode 100644
index 0000000..1ccea1e
--- /dev/null
+++ b/linux/system/profile.sls
@@ -0,0 +1,35 @@
+{%- from "linux/map.jinja" import system with context %}
+{%- if system.enabled %}
+
+/etc/profile.d:
+ file.directory:
+ - user: root
+ - mode: 750
+ - makedirs: true
+
+profile.d_clean:
+ file.directory:
+ - name: /etc/profile.d
+ - clean: true
+ - exclude_pat: 'E@^((?!salt_profile*).)*$'
+
+{%- if system.profile|length > 0 %}
+
+{%- for name, script in system.profile.iteritems() %}
+profile.d_script_{{ name }}:
+ file.managed:
+ - name: /etc/profile.d/salt_profile_{{ name }}{%if name.split('.')|length == 1 %}.sh{% endif %}
+ - mode: 755
+ - source:
+ - salt://linux/files/etc_profile_{{ name }}
+ - salt://linux/files/etc_profile
+ - template: jinja
+ - defaults:
+ script: {{ script|yaml }}
+ - require_in:
+ - service: profile.d_clean
+{% endfor %}
+
+{%- endif %}
+{%- endif %}
+
diff --git a/linux/system/repo.sls b/linux/system/repo.sls
index 555abd1..5a4dce7 100644
--- a/linux/system/repo.sls
+++ b/linux/system/repo.sls
@@ -1,12 +1,51 @@
{%- from "linux/map.jinja" import system with context %}
{%- if system.enabled %}
+# global proxy setup
+{%- if system.proxy.get('pkg', {}).get('enabled', False) %}
+{%- if grains.os_family == 'Debian' %}
+
+/etc/apt/apt.conf.d/99proxies-salt:
+ file.managed:
+ - template: jinja
+ - source: salt://linux/files/apt.conf.d_proxies
+ - defaults:
+ external_host: False
+ https: {{ system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true) }}
+ http: {{ system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true) }}
+ ftp: {{ system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true) }}
+
+{%- else %}
+
+/etc/apt/apt.conf.d/99proxies-salt:
+ file.absent
+
+{%- endif %}
+{%- endif %}
+
{% set default_repos = {} %}
{%- for name, repo in system.repo.iteritems() %}
{%- if grains.os_family == 'Debian' %}
+# per repository proxy setup
+{%- if repo.get('proxy', {}).get('enabled', False) %}
+{%- set external_host = repo.proxy.get('host', None) or repo.source.split('/')[2] %}
+/etc/apt/apt.conf.d/99proxies-salt-{{ name }}:
+ file.managed:
+ - template: jinja
+ - source: salt://linux/files/apt.conf.d_proxies
+ - defaults:
+ external_host: {{ external_host }}
+ https: {{ repo.proxy.get('https', None) or system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), True) }}
+ http: {{ repo.proxy.get('http', None) or system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), True) }}
+ ftp: {{ repo.proxy.get('ftp', None) or system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), True) }}
+{%- else %}
+/etc/apt/apt.conf.d/99proxies-salt-{{ name }}:
+ file.absent
+{%- endif %}
+
{%- if repo.pin is defined %}
linux_repo_{{ name }}_pin:
@@ -58,8 +97,17 @@
{%- if repo.key_url is defined %}
- key_url: {{ repo.key_url }}
{%- endif %}
+ - consolidate: {{ repo.get('consolidate', False) }}
+ - clean_file: {{ repo.get('clean_file', False) }}
+ - refresh_db: {{ repo.get('refresh_db', True) }}
- require:
- pkg: linux_packages
+ {%- if repo.get('proxy', {}).get('enabled', False) %}
+ - file: /etc/apt/apt.conf.d/99proxies-salt-{{ name }}
+ {%- endif %}
+ {%- if system.proxy.get('pkg', {}).get('enabled', False) %}
+ - file: /etc/apt/apt.conf.d/99proxies-salt
+ {%- endif %}
{%- endif %}
@@ -67,6 +115,11 @@
{%- if grains.os_family == "RedHat" %}
+{%- if repo.get('proxy', {}).get('enabled', False) %}
+# PLACEHOLDER
+# TODO, implement per proxy configuration for Yum
+{%- endif %}
+
{%- if not repo.get('default', False) %}
linux_repo_{{ name }}:
diff --git a/tests/integration/system/env_spec.rb b/tests/integration/system/env_spec.rb
new file mode 100644
index 0000000..2f5c689
--- /dev/null
+++ b/tests/integration/system/env_spec.rb
@@ -0,0 +1,12 @@
+
+## PROXIES
+#
+describe file('/etc/environment') do
+ it('should exist')
+ its('content') { should_not match /HTTPS_PROXY"/ }
+ its('content') { should match /HTTP_PROXY="http:\/\/127.0.4.2:80"/ }
+ its('content') { should match /BOB_PATH=/}
+ its('content') { should match /LC_ALL="C"/ }
+ its('content') { should match /ftp_proxy=.*127.0.4.3:2121/ }
+ its('content') { should match /NO_PROXY=.*dummy.net,.local/ }
+end
diff --git a/tests/integration/system/profile_spec.rb b/tests/integration/system/profile_spec.rb
new file mode 100644
index 0000000..d51c11e
--- /dev/null
+++ b/tests/integration/system/profile_spec.rb
@@ -0,0 +1,17 @@
+
+describe file('/etc/profile.d/salt_profile_vi_flavors.sh') do
+ it('should exist')
+ its('content') { should match /EDITOR=vim/ }
+ its('content') { should match /PAGER=view/ }
+ its('content') { should match /alias vi=vim/ }
+end
+
+describe file('/etc/profile.d/salt_profile_locales.sh') do
+ it('should exist')
+ its('content') { should match /LANG=en_US/ }
+end
+
+describe file('/etc/profile.d/prompt.sh') do
+ it('should exist')
+end
+
diff --git a/tests/integration/system/repo_spec.rb b/tests/integration/system/repo_spec.rb
new file mode 100644
index 0000000..91b4b9b
--- /dev/null
+++ b/tests/integration/system/repo_spec.rb
@@ -0,0 +1,17 @@
+
+# PROXIES
+#
+# globally
+describe file('/etc/apt/apt.conf.d/99proxies-salt') do
+ it('should exist')
+ its('content') { should_not match /ftp/ }
+ its('content') { should match /proxy "https.*127.0.2.1:4443"/ }
+end
+
+# per repo
+describe file('/etc/apt/apt.conf.d/99proxies-salt-opencontrail') do
+ it('should exist')
+ its('content') { should_not match /ftp/ }
+ its('content') { should match /Acquire::https::proxy::ppa.launchpad.net/ }
+end
+
diff --git a/tests/pillar/system.sls b/tests/pillar/system.sls
index d92dc8e..216bca5 100644
--- a/tests/pillar/system.sls
+++ b/tests/pillar/system.sls
@@ -90,7 +90,29 @@
version: latest
repo:
opencontrail:
- source: "deb http://ppa.launchpad.net/tcpcloud/contrail-2.20/ubuntu trusty main"
+ source: "deb http://ppa.launchpad.net/tcpcloud/contrail-3.0/ubuntu xenial main"
+ keyid: E79EE90C
+ keyserver: keyserver.ubuntu.com
+ architectures: amd64
+ proxy:
+ enabled: true
+ https: https://127.0.5.1:443
+ #http: http://127.0.5.2:8080
+ apt-mk-salt:
+ source: "deb http://apt-mk.mirantis.com/xenial stable salt"
+ key_url: http://apt-mk.mirantis.com/public.gpg
+ architectures: amd64
+ proxy:
+ enabled: true
+ apt-mk-salt-nightly:
+ source: "deb http://apt-mk.mirantis.com/xenial nightly salt"
+ key_url: http://apt-mk.mirantis.com/public.gpg
+ architectures: amd64
+ proxy:
+ enabled: false
+ apt-mk-extra-nightly:
+ source: "deb http://apt-mk.mirantis.com/xenial nightly extra"
+ key_url: http://apt-mk.mirantis.com/public.gpg
architectures: amd64
locale:
en_US.UTF-8:
@@ -200,3 +222,46 @@
sudogroup3:
commands:
- ALL
+ env:
+ BOB_VARIABLE: Alice
+ BOB_PATH:
+ - /srv/alice/bin
+ - /srv/bob/bin
+ HTTPS_PROXY: https://127.0.4.1:443
+ http_proxy: http://127.0.4.2:80
+ ftp_proxy: ftp://127.0.4.3:2121
+ no_proxy:
+ - 192.168.0.1
+ - 192.168.0.2
+ - .saltstack.com
+ - .ubuntu.com
+ - .mirantis.com
+ - .launchpad.net
+ - .dummy.net
+ - .local
+ LANG: C
+ LC_ALL: C
+ profile:
+ vi_flavors.sh: |
+ export PAGER=view
+ alias vi=vim
+ locales: |
+ export LANG=en_US
+ export LC_ALL=en_US.UTF-8
+
+ # pillar for proxy configuration
+ proxy:
+ # for package managers
+ pkg:
+ enabled: true
+ https: https://127.0.2.1:4443
+ #http: http://127.0.2.2
+ ftp: none
+ # fallback, system defaults
+ https: https://127.0.1.1:443
+ #http: http://127.0.1.2
+ ftp: ftp://127.0.1.3
+ noproxy:
+ - host1
+ - host2
+ - .local