Merge pull request #88 from salt-formulas/pr/preinstall-apt-https-tranpsort
preinstall apt https transport
diff --git a/.travis.yml b/.travis.yml
index 9ab7afc..577330b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,18 +18,29 @@
gem 'kitchen-docker'
gem 'kitchen-inspec'
gem 'inspec'
- gem 'kitchen-salt', :git => 'https://github.com/epcim/kitchen-salt.git', :branch => 'dependencis-pkg-repo2'
- #Waiting for PR#78
- #gem 'kitchen-salt', '>=0.2.25'
+ gem 'kitchen-salt', :git => 'https://github.com/salt-formulas/kitchen-salt.git'
- bundle install
env:
- - PLATFORM=trevorj/salty-whales:trusty
- - PLATFORM=trevorj/salty-whales:xenial
+ - PLATFORM=trevorj/salty-whales:trusty
+ - PLATFORM=trevorj/salty-whales:xenial
+
before_script:
+ - set -o pipefail
- make test | tail
script:
- test ! -e .kitchen.yml || bundle exec kitchen converge || true
- test ! -e .kitchen.yml || bundle exec kitchen verify -t tests/integration
+
+notifications:
+ webhooks:
+ urls:
+ - https://webhooks.gitter.im/e/6123573504759330786b
+ on_success: change # options: [always|never|change] default: always
+ on_failure: never # options: [always|never|change] default: always
+ on_start: never # options: [always|never|change] default: always
+ on_cancel: never # options: [always|never|change] default: always
+ on_error: never # options: [always|never|change] default: always
+ email: false
diff --git a/README.rst b/README.rst
index f9dede9..856851b 100644
--- a/README.rst
+++ b/README.rst
@@ -1,6 +1,6 @@
-=====
-Linux
-=====
+============
+Linux Fomula
+============
Linux Operating Systems.
@@ -10,10 +10,11 @@
* Fedora
* Arch
-Sample pillars
+Sample Pillars
==============
-Linux system
+
+Linux System
------------
Basic Linux box
@@ -57,6 +58,7 @@
This ways ``linux.system.sudo`` pillar map to actual sudo attributes:
.. code-block:: jinja
+
# simplified template:
Cmds_Alias {{ alias }}={{ commands }}
{{ user }} {{ hosts }}=({{ runas }}) NOPASSWD: {{ commands }}
@@ -65,8 +67,8 @@
# when rendered:
saltuser1 ALL=(ALL) NOPASSWD: ALL
-
.. code-block:: yaml
+
linux:
system:
sudo:
@@ -888,6 +890,14 @@
- timeout: 2
- attempts: 2
+**setting custom TX queue length for tap interfaces**
+
+.. code-block:: yaml
+
+ linux:
+ network:
+ tap_custom_txqueuelen: 10000
+
DPDK OVS interfaces
--------------------
@@ -998,6 +1008,22 @@
- file_system: cifs
- options: guest,uid=myuser,iocharset=utf8,file_mode=0777,dir_mode=0777,noperm
+NFS mount
+
+.. code-block:: yaml
+
+ linux:
+ storage:
+ enabled: true
+ mount:
+ nfs_glance:
+ enabled: true
+ path: /var/lib/glance/images
+ device: 172.16.10.110:/var/nfs/glance
+ file_system: nfs
+ opts: rw,sync
+
+
File swap configuration
.. code-block:: yaml
@@ -1160,6 +1186,32 @@
username: test
password: test
+Netconsole Remote Kernel Logging
+--------------------------------
+
+Netconsole logger could be configured for configfs-enabled kernels
+(`CONFIG_NETCONSOLE_DYNAMIC` should be enabled). Configuration applies both in
+runtime (if network is already configured), and on-boot after interface
+initialization. Notes:
+
+ * receiver could be located only in same L3 domain
+ (or you need to configure gateway MAC manually)
+ * receiver's MAC is detected only on configuration time
+ * using broadcast MAC is not recommended
+
+.. code-block:: yaml
+
+ parameters:
+ linux:
+ system:
+ netconsole:
+ enabled: true
+ port: 514 (optional)
+ loglevel: debug (optional)
+ target:
+ 192.168.0.1:
+ interface: bond0
+ mac: "ff:ff:ff:ff:ff:ff" (optional)
Usage
=====
diff --git a/linux/files/60-net-txqueue.rules b/linux/files/60-net-txqueue.rules
new file mode 100644
index 0000000..7805c9c
--- /dev/null
+++ b/linux/files/60-net-txqueue.rules
@@ -0,0 +1 @@
+KERNEL==”tap[0-9a-z\-]*", RUN+="/sbin/ip link set %k txqueuelen {{ network.tap_custom_txqueuelen }}"
\ No newline at end of file
diff --git a/linux/files/etc_environment b/linux/files/etc_environment
index 3b6f480..0ffa7e1 100644
--- a/linux/files/etc_environment
+++ b/linux/files/etc_environment
@@ -11,22 +11,22 @@
{%- endfor %}
{%- if ftp_proxy and ftp_proxy.lower() != 'none' %}
-ftp_proxy="{{ ftp_proxy }}";
-FTP_PROXY="{{ ftp_proxy }}";
+ftp_proxy="{{ ftp_proxy }}"
+FTP_PROXY="{{ ftp_proxy }}"
{%- endif %}
{%- if http_proxy and http_proxy.lower() != 'none' %}
-http_proxy="{{ http_proxy }}";
-HTTP_PROXY="{{ http_proxy }}";
+http_proxy="{{ http_proxy }}"
+HTTP_PROXY="{{ http_proxy }}"
{%- endif %}
{%- if https_proxy and https_proxy.lower() != 'none' %}
-https_proxy="{{ https_proxy }}";
-HTTPS_PROXY="{{ https_proxy }}";
+https_proxy="{{ https_proxy }}"
+HTTPS_PROXY="{{ https_proxy }}"
{%- endif %}
{%- if no_proxy %}
-no_proxy="{{ no_proxy|join(',') }}";
-NO_PROXY="{{ no_proxy|join(',') }}";
+no_proxy="{{ no_proxy|join(',') }}"
+NO_PROXY="{{ no_proxy|join(',') }}"
{%- endif %}
diff --git a/linux/files/netconsole b/linux/files/netconsole
new file mode 100644
index 0000000..d7e1e67
--- /dev/null
+++ b/linux/files/netconsole
@@ -0,0 +1,136 @@
+#!/bin/sh
+SYSFS_NETCONSOLE="/sys/kernel/config/netconsole"
+NETCONSOLE_CONF="/etc/default/netconsole.conf"
+NETCONSOLE_PORT="514"
+
+netconsole_remove() {
+ for sysfsnc in "${SYSFS_NETCONSOLE}/${interface:-}-"*
+ do
+ if [ -e "${sysfsnc}" ]
+ then
+ logger -t netconsole "remove ${sysfsnc}"
+ rmdir "${sysfsnc}"
+ fi
+ done
+}
+
+netconsole_remote_mac()
+{
+ neigh()
+ {
+ ip -4 -o neigh show to "${remote_ip}" dev "${interface}" | cut -d\ -f3
+ }
+ remote_mac="$(neigh)"
+ if [ -n "${remote_mac:-}" ] && [ "${remote_mac:-}" != "INCOMPLETE" ]
+ then
+ if [ "${remote_mac:-}" != "FAILED" ]
+ then
+ echo "${remote_mac:-}"
+ return 0
+ fi
+ else
+ if ping -n -q -c 1 -w 1 -I "${interface}" "${remote_ip}" >/dev/null && remote_mac="$(neigh)" && [ -n "${remote_mac:-}" ]
+ then
+ echo "${remote_mac:-}"
+ return 0
+ fi
+ fi
+ return 1
+}
+
+netconsole_add() {
+ netconsole() {
+ iface="${1:-}"
+ remote_ip="${2:-}"
+ remote_mac="${3:-}"
+
+ if [ "${iface:-}" = "${interface:-}" ] && [ -n "${remote_ip:-}" ]
+ then
+ logger -t netconsole "from ${new_ip_address:-}@${interface:-}"
+ else
+ return 1
+ fi
+ if [ -n "${remote_mac}" ] || remote_mac="$(netconsole_remote_mac)"
+ then
+ logger -t netconsole "to ${remote_ip} ${remote_mac}"
+ else
+ return 1
+ fi
+
+ sysfsnc="${SYSFS_NETCONSOLE}/${interface}-${remote_ip}"
+
+ if [ -e "${sysfsnc}" ] && [ -z "${old_ip_address:-}" ]
+ then
+ old_ip_address="$(cat "${sysfsnc}/local_ip")"
+ fi
+
+ if [ "${old_ip_address:-}" != "${new_ip_address:-}" ] || ! [ -e "${sysfsnc}" ]
+ then
+ logger -t netconsole "setup netconsole"
+ else
+ return 1
+ fi
+
+ mkdir -p "${sysfsnc}"
+ if [ "$(cat "${sysfsnc}/enabled")" != "0" ]
+ then
+ echo "0" > "${sysfsnc}/enabled"
+ fi
+
+ if [ -n "${new_ip_address:-}" ]
+ then
+ echo "${new_ip_address}" > "${sysfsnc}/local_ip"
+ fi
+ echo "${interface}" > "${sysfsnc}/dev_name"
+ echo "${remote_mac}" > "${sysfsnc}/remote_mac"
+ echo "${remote_ip}" > "${sysfsnc}/remote_ip"
+ echo "${PORT:-${NETCONSOLE_PORT}}" > "${sysfsnc}/remote_port"
+ echo "1" > "${sysfsnc}/enabled"
+ return 0
+ }
+
+ if [ -f "${NETCONSOLE_CONF}" ]
+ then
+ modprobe netconsole
+ mountpoint -q /sys/kernel/config || mount none -t configfs /sys/kernel/config
+
+ if [ -e "${SYSFS_NETCONSOLE}" ]
+ then
+ (
+ set -x
+ set +e
+ . "${NETCONSOLE_CONF}"
+ ) ||:
+ fi
+ fi
+}
+
+netconsole_setup() {
+ case ${reason:-} in
+ BOUND|RENEW|REBIND|REBOOT)
+ netconsole_add
+ ;;
+ EXPIRE|FAIL|RELEASE|STOP)
+ netconsole_remove
+ ;;
+ PREINIT) : ;;
+ *)
+ if [ "${ADDRFAM:-}" = "inet" ] && [ "${METHOD:-}" = "static" ]
+ then
+ export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ interface="${IFACE:-}"
+ new_ip_address="${IF_ADDRESS:-}"
+ case ${MODE:-} in
+ start)
+ netconsole_add
+ ;;
+ stop)
+ netconsole_remove
+ ;;
+ *) : ;;
+ esac
+ fi
+ esac
+}
+
+netconsole_setup
diff --git a/linux/files/netconsole.conf b/linux/files/netconsole.conf
new file mode 100644
index 0000000..01b2ced
--- /dev/null
+++ b/linux/files/netconsole.conf
@@ -0,0 +1,22 @@
+{%- from "linux/map.jinja" import system with context %}
+# default port is 514
+#PORT=6666
+{%- if system.netconsole is mapping and system.netconsole.port is defined %}
+PORT="{{ system.netconsole.port }}"
+{%- endif %}
+
+# unicast, could be multiline
+#netconsole ens3 192.168.1.32 fa:16:3e:8d:f6:d0
+{%- if system.netconsole is mapping and system.netconsole.target is mapping %}
+{%- for target, data in system.netconsole.target.iteritems() %}
+{%- if data is mapping %}
+netconsole "{{ data.get('interface', '${interface}') }}" "{{ target }}" "{{ data.get('mac', '') }}"
+{%- endif %}
+{%- endfor %}
+{%- endif %}
+
+# set up dmesg log level
+# dmesg -n info
+{%- if system.netconsole is mapping and system.netconsole.loglevel is defined %}
+dmesg -n "{{ system.netconsole.loglevel }}"
+{%- endif %}
diff --git a/linux/map.jinja b/linux/map.jinja
index 25eefae..045212d 100644
--- a/linux/map.jinja
+++ b/linux/map.jinja
@@ -146,6 +146,9 @@
'swap': {},
'lvm': {},
'loopback': {},
+ 'nfs': {
+ 'pkgs': ['nfs-utils']
+ },
'multipath': {
'enabled': False,
'pkgs': ['multipath-tools', 'multipath-tools-boot'],
@@ -157,6 +160,9 @@
'swap': {},
'lvm': {},
'loopback': {},
+ 'nfs': {
+ 'pkgs': ['nfs-common']
+ },
'multipath': {
'enabled': False,
'pkgs': ['multipath-tools', 'multipath-tools-boot'],
@@ -169,6 +175,9 @@
'swap': {},
'lvm': {},
'loopback': {},
+ 'nfs': {
+ 'pkgs': ['nfs-utils']
+ },
'multipath': {
'enabled': False,
'pkgs': [],
diff --git a/linux/meta/prometheus.yml b/linux/meta/prometheus.yml
new file mode 100644
index 0000000..83e8eac
--- /dev/null
+++ b/linux/meta/prometheus.yml
@@ -0,0 +1,44 @@
+{% raw %}
+server:
+ alert:
+ AvgCPUUsageIdle:
+ if: 'avg_over_time(cpu_usage_idle{cpu="cpu-total"}[5m]) < 10'
+ labels:
+ severity: warning
+ service: system
+ annotations:
+ summary: 'Avarage CPU usage (idle) for node {{ $labels.host }} is low'
+ description: 'Avarage CPU usage (idle) for node {{ $labels.host }} is low {{ $value }}'
+ PredictLinearDiskFree:
+ if: 'predict_linear(disk_free[1h], 8*3600) < 0'
+ labels:
+ severity: warning
+ service: system
+ annotations:
+ summary: 'Disk space ({{ $labels.path }}) is filling on {{ $labels.host }}'
+ description: 'Disk space ({{ $labels.path }}) will be full in 8h on {{ $labels.host }}'
+ PredictLinearDiskInodesFree:
+ if: 'predict_linear(disk_inodes_free[1h], 8*3600) < 0'
+ labels:
+ severity: warning
+ service: system
+ annotations:
+ summary: 'Disk inodes ({{ $labels.path }}) are filling on {{ $labels.host }}'
+ description: 'Disk inodes ({{ $labels.path }}) will be full in 8h on {{ $labels.host }}'
+ AvgMemAvailablePercent:
+ if: 'avg_over_time(mem_available_percent[5m]) < 10'
+ labels:
+ severity: warning
+ service: system
+ annotations:
+ summary: 'Free memory is low on {{ $labels.host }}'
+ description: 'Free memory percent for node {{ $labels.host }} is low {{ $value }}'
+ SystemLoad5:
+ if: 'system_load5 / system_n_cpus > 3'
+ labels:
+ severity: warning
+ service: system
+ annotations:
+ summary: 'High system load (5m) on {{ $labels.host }}'
+ description: 'High system load (5m) on node {{ $labels.host }}'
+{% endraw %}
diff --git a/linux/meta/telegraf.yml b/linux/meta/telegraf.yml
new file mode 100644
index 0000000..1cd5277
--- /dev/null
+++ b/linux/meta/telegraf.yml
@@ -0,0 +1,8 @@
+agent:
+ input:
+ cpu:
+ percpu: true
+ totalcpu: true
+ disk:
+ mem:
+ system:
diff --git a/linux/network/interface.sls b/linux/network/interface.sls
index ad695ae..bed9a3f 100644
--- a/linux/network/interface.sls
+++ b/linux/network/interface.sls
@@ -292,3 +292,13 @@
- enable: false
{%- endif %}
+
+{%- if network.tap_custom_txqueuelen is defined %}
+
+/etc/udev/rules.d/60-net-txqueue.rules:
+ file.managed:
+ - source: salt://linux/files/60-net-txqueue.rules
+ - mode: 755
+ - template: jinja
+
+{%- endif %}
diff --git a/linux/storage/mount.sls b/linux/storage/mount.sls
index e4c2166..465f966 100644
--- a/linux/storage/mount.sls
+++ b/linux/storage/mount.sls
@@ -24,6 +24,12 @@
{%- endif %}
+{%- if mount.file_system == 'nfs' %}
+linux_storage_nfs_packages:
+ pkg.installed:
+ - pkgs: {{ storage.nfs.pkgs }}
+{%- endif %}
+
{{ mount.path }}:
mount.mounted:
- device: {{ mount.device }}
diff --git a/linux/system/init.sls b/linux/system/init.sls
index 84c2365..45700f0 100644
--- a/linux/system/init.sls
+++ b/linux/system/init.sls
@@ -80,3 +80,6 @@
{%- if system.sudo is defined %}
- linux.system.sudo
{%- endif %}
+{%- if system.netconsole is defined %}
+- linux.system.netconsole
+{%- endif %}
diff --git a/linux/system/kernel.sls b/linux/system/kernel.sls
index 11d5a38..0c352ef 100644
--- a/linux/system/kernel.sls
+++ b/linux/system/kernel.sls
@@ -10,7 +10,7 @@
/etc/default/grub.d/90-isolcpu.cfg:
file.managed:
- - contents: 'GRUB_CMDLINE_LINUX_DEFAULT="$GRUB_CMDLINE_LINUX_DEFAULT isolcpu={{ system.kernel.isolcpu }}"'
+ - contents: 'GRUB_CMDLINE_LINUX_DEFAULT="$GRUB_CMDLINE_LINUX_DEFAULT isolcpus={{ system.kernel.isolcpu }}"'
- require:
- file: grub_d_directory
- watch_in:
diff --git a/linux/system/netconsole.sls b/linux/system/netconsole.sls
new file mode 100644
index 0000000..92dab5d
--- /dev/null
+++ b/linux/system/netconsole.sls
@@ -0,0 +1,45 @@
+{% from "linux/map.jinja" import system with context %}
+{% if system.enabled and system.netconsole is mapping and system.netconsole.enabled %}
+
+/etc/dhcp/dhclient-exit-hooks.d/netconsole:
+ file.managed:
+ - source: salt://linux/files/netconsole
+ - makedirs: True
+
+/etc/network/if-up.d/netconsole:
+ file.managed:
+ - source: salt://linux/files/netconsole
+ - mode: 755
+ - makedirs: True
+
+/etc/network/if-down.d/netconsole:
+ file.managed:
+ - source: salt://linux/files/netconsole
+ - mode: 755
+ - makedirs: True
+
+/etc/default/netconsole.conf:
+ file.managed:
+ - source: salt://linux/files/netconsole.conf
+ - template: jinja
+
+{% if system.netconsole is mapping and system.netconsole.target is mapping %}
+{% for target, data in system.netconsole.target.iteritems() %}
+{% if data is mapping and data.interface is defined %}
+/etc/network/if-up.d/netconsole {{ target }} {{ data.interface }}:
+ cmd.run:
+ - name: /etc/network/if-up.d/netconsole
+ - env:
+ - IFACE: {{ data.interface }}
+ - METHOD: static
+ - ADDRFAM: inet
+ - MODE: start
+ - onchanges:
+ - file: /etc/default/netconsole.conf
+ - require:
+ - file: /etc/network/if-up.d/netconsole
+{% endif %}
+{% endfor %}
+{% endif %}
+
+{% endif %}
diff --git a/linux/system/profile.sls b/linux/system/profile.sls
index 1ccea1e..7e1fb28 100644
--- a/linux/system/profile.sls
+++ b/linux/system/profile.sls
@@ -1,12 +1,6 @@
{%- from "linux/map.jinja" import system with context %}
{%- if system.enabled %}
-/etc/profile.d:
- file.directory:
- - user: root
- - mode: 750
- - makedirs: true
-
profile.d_clean:
file.directory:
- name: /etc/profile.d
@@ -19,7 +13,6 @@
profile.d_script_{{ name }}:
file.managed:
- name: /etc/profile.d/salt_profile_{{ name }}{%if name.split('.')|length == 1 %}.sh{% endif %}
- - mode: 755
- source:
- salt://linux/files/etc_profile_{{ name }}
- salt://linux/files/etc_profile
diff --git a/linux/system/repo.sls b/linux/system/repo.sls
index 5a4dce7..408fe98 100644
--- a/linux/system/repo.sls
+++ b/linux/system/repo.sls
@@ -1,6 +1,9 @@
{%- from "linux/map.jinja" import system with context %}
{%- if system.enabled %}
+include:
+- linux.system.package
+
# global proxy setup
{%- if system.proxy.get('pkg', {}).get('enabled', False) %}
{%- if grains.os_family == 'Debian' %}
@@ -25,6 +28,15 @@
{% set default_repos = {} %}
+{%- if system.purge_repos|default(False) %}
+
+purge_sources_list_d_repos:
+ file.directory:
+ - name: /etc/apt/sources.list.d/
+ - clean: True
+
+{%- endif %}
+
{%- for name, repo in system.repo.iteritems() %}
{%- if grains.os_family == 'Debian' %}
@@ -88,6 +100,7 @@
- architectures: {{ repo.architectures }}
{%- endif %}
- file: /etc/apt/sources.list.d/{{ name }}.list
+ - clean_file: {{ repo.clean|default(True) }}
{%- if repo.key_id is defined %}
- keyid: {{ repo.key_id }}
{%- endif %}
@@ -108,6 +121,9 @@
{%- if system.proxy.get('pkg', {}).get('enabled', False) %}
- file: /etc/apt/apt.conf.d/99proxies-salt
{%- endif %}
+ {%- if system.purge_repos|default(False) %}
+ - file: purge_sources_list_d_repos
+ {%- endif %}
{%- endif %}
@@ -154,6 +170,9 @@
- user: root
- group: root
- mode: 0644
+{%- if system.purge_repos %}
+ - replace: True
+{%- endif %}
- defaults:
default_repos: {{ default_repos }}
- require:
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index 88b7f2c..7699edd 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -1,6 +1,10 @@
parameters:
linux:
_support:
+ prometheus:
+ enabled: true
+ telegraf:
+ enabled: true
collectd:
enabled: true
heka:
diff --git a/metadata/service/system/init.yml b/metadata/service/system/init.yml
index 2c4cc9c..e75718a 100644
--- a/metadata/service/system/init.yml
+++ b/metadata/service/system/init.yml
@@ -13,6 +13,7 @@
home: /root
timezone: Europe/Prague
cluster: default
+ purge_repos: false
network:
enabled: true
hostname: ${linux:system:name}
diff --git a/tests/integration/system/netconsole_spec.rb b/tests/integration/system/netconsole_spec.rb
new file mode 100644
index 0000000..f517508
--- /dev/null
+++ b/tests/integration/system/netconsole_spec.rb
@@ -0,0 +1,14 @@
+
+## NETCONSOLE
+#
+describe file('/etc/default/netconsole.conf') do
+ it('should exist')
+ its('content') { should match /^PORT="514"/}
+ its('content') { should match /^netconsole "bond0" "192.168.0.1" "ff:ff:ff:ff:ff:ff"/}
+ its('content') { should match /^dmesg -n "debug"/}
+end
+
+describe file('/etc/dhcp/dhclient-exit-hooks.d/netconsole') do
+ it('should exist')
+ its('content') { should match /netconsole_setup/}
+end
diff --git a/tests/pillar/system.sls b/tests/pillar/system.sls
index 216bca5..4775253 100644
--- a/tests/pillar/system.sls
+++ b/tests/pillar/system.sls
@@ -6,6 +6,7 @@
domain: local
environment: prd
hostname: system.pillar.local
+ purge_repos: true
apparmor:
enabled: false
haveged:
@@ -265,3 +266,13 @@
- host1
- host2
- .local
+
+ # pillars for netconsole setup
+ netconsole:
+ enabled: true
+ port: 514
+ loglevel: debug
+ target:
+ 192.168.0.1:
+ mac: "ff:ff:ff:ff:ff:ff"
+ interface: bond0