Add apparmor support for virtlet
Related story: https://mirantis.jira.com/browse/PROD-22569
Change-Id: I53b20bdf6753d34a86518ce454e6243ddfd57854
diff --git a/kubernetes/files/kube-addons/virtlet/apparmor/copyright b/kubernetes/files/kube-addons/virtlet/apparmor/copyright
new file mode 100644
index 0000000..8a227b1
--- /dev/null
+++ b/kubernetes/files/kube-addons/virtlet/apparmor/copyright
@@ -0,0 +1,67 @@
+libvirt-qemu:
+
+ This profile is a part of libvirt-bin:1.3.1-1ubuntu10.24 package
+
+ libvirt was initially debianized by Andrew Mitchell <ajmitch@ubuntu.com>
+
+ It was downloaded from http://libvirt.org/sources/
+
+ Upstream Author:
+
+ Alex Jia <ajia@redhat.com>
+ Anthony Liguori <aliguori@us.ibm.com>
+ Chris Lalancette <clalance@redhat.com>
+ Christophe Fergeau <cfergeau@redhat.com>
+ Claudio Bley <cbley@av-test.de>
+ Cole Robinson <crobinso@redhat.com>
+ Daniel Berrange <berrange@redhat.com>
+ Daniel Veillard <veillard@redhat.com>
+ Dave Allan <dallan@redhat.com>
+ Doug Goldstein <cardoe@gentoo.org>
+ Eric Blake <eblake@redhat.com>
+ Gao Feng <gaofeng@cn.fujitsu.com>
+ Guannan Ren <gren@redhat.com>
+ Guido Günther <agx@sigxcpu.org>
+ Ján Tomko <jtomko@redhat.com>
+ Jim Fehlig <jfehlig@suse.com>
+ Jim Meyering <meyering@redhat.com>
+ Jiří Denemark <jdenemar@redhat.com>
+ John Ferlan <jferlan@redhat.com>
+ John Levon <john.levon@sun.com>
+ Laine Stump <laine@redhat.com>
+ Mark McLoughlin <markmc@redhat.com>
+ Martin Kletzander <mkletzan@redhat.com>
+ Matthias Bolte <matthias.bolte@googlemail.com>
+ Michal Prívozník <mprivozn@redhat.com>
+ Osier Yang <jyang@redhat.com>
+ Pavel Hrdina <phrdina@redhat.com>
+ Peter Krempa <pkrempa@redhat.com>
+ Richard W.M. Jones <rjones@redhat.com>
+ Roman Bogorodskiy <bogorodskiy@gmail.com>
+ Stefan Berger <stefanb@us.ibm.com>
+ Wen Congyang <wency@cn.fujitsu.com>
+ Atsushi SAKAI <sakaia@jp.fujitsu.com>
+ Dan Smith <danms@us.ibm.com>
+ Dave Leskovec <dlesko@linux.vnet.ibm.com>
+ Justin Clift <jclift@redhat.com>
+ Karel Zak <kzak@redhat.com>
+
+ Copyright:
+
+ 2005-2014 Red Hat, Inc
+
+ Licenses:
+
+ This library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ This library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this library; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
diff --git a/kubernetes/files/kube-addons/virtlet/apparmor/libvirt-qemu b/kubernetes/files/kube-addons/virtlet/apparmor/libvirt-qemu
new file mode 100644
index 0000000..595cccd
--- /dev/null
+++ b/kubernetes/files/kube-addons/virtlet/apparmor/libvirt-qemu
@@ -0,0 +1,228 @@
+# Last Modified: Wed Jul 8 09:57:41 2009
+
+ #include <abstractions/base>
+ #include <abstractions/consoles>
+ #include <abstractions/nameservice>
+
+ # required for reading disk images
+ capability dac_override,
+ capability dac_read_search,
+ capability chown,
+
+ # needed to drop privileges
+ capability setgid,
+ capability setuid,
+
+ # this is needed with libcap-ng support, however it breaks a lot of things
+ # atm, so just silence the denial until libcap-ng works right. LP: #522845
+ deny capability setpcap,
+
+ network inet stream,
+ network inet6 stream,
+
+ /dev/net/tun rw,
+ /dev/tap* rw,
+ /dev/kvm rw,
+ /dev/ptmx rw,
+ /dev/kqemu rw,
+ @{PROC}/*/status r,
+ @{PROC}/sys/kernel/cap_last_cap r,
+ owner @{PROC}/*/auxv r,
+ @{PROC}/sys/vm/overcommit_memory r,
+
+ /sys/devices/system/node/ r,
+ /sys/devices/system/node/node[0-9]*/meminfo r,
+ /sys/devices/system/cpu/ r,
+
+ /sys/module/vhost/parameters/max_mem_regions r,
+
+ # For hostdev access. The actual devices will be added dynamically
+ /sys/bus/usb/devices/ r,
+ /sys/devices/**/usb[0-9]*/** r,
+
+ # WARNING: this gives the guest direct access to host hardware and specific
+ # portions of shared memory. This is required for sound using ALSA with kvm,
+ # but may constitute a security risk. If your environment does not require
+ # the use of sound in your VMs, feel free to comment out or prepend 'deny' to
+ # the rules for files in /dev.
+ /{dev,run}/shm r,
+ /{dev,run}/shmpulse-shm* r,
+ /{dev,run}/shmpulse-shm* rwk,
+ /dev/snd/* rw,
+ capability ipc_lock,
+ # spice
+ /usr/bin/qemu-system-i386-spice rmix,
+ /usr/bin/qemu-system-x86_64-spice rmix,
+ /{dev,run}/shm/ r,
+ owner /{dev,run}/shm/spice.* rw,
+ # 'kill' is not required for sound and is a security risk. Do not enable
+ # unless you absolutely need it.
+ deny capability kill,
+
+ # Uncomment the following if you need access to /dev/fb*
+ #/dev/fb* rw,
+
+ /etc/pulse/client.conf r,
+ @{HOME}/.pulse-cookie rwk,
+ owner /root/.pulse-cookie rwk,
+ owner /root/.pulse/ rw,
+ owner /root/.pulse/* rw,
+ /usr/share/alsa/** r,
+ owner /tmp/pulse-*/ rw,
+ owner /tmp/pulse-*/* rw,
+ /var/lib/dbus/machine-id r,
+
+ # access to firmware's etc
+ /usr/share/kvm/** r,
+ /usr/share/qemu/** r,
+ /usr/share/bochs/** r,
+ /usr/share/openbios/** r,
+ /usr/share/openhackware/** r,
+ /usr/share/proll/** r,
+ /usr/share/vgabios/** r,
+ /usr/share/seabios/** r,
+ /usr/share/misc/sgabios.bin r,
+ /usr/share/ovmf/** r,
+ /usr/share/slof/** r,
+
+ # access PKI infrastructure
+ /etc/pki/libvirt-vnc/** r,
+
+ # the various binaries
+ /usr/bin/kvm rmix,
+ /usr/bin/qemu rmix,
+ /usr/bin/qemu-system-aarch64 rmix,
+ /usr/bin/qemu-system-alpha rmix,
+ /usr/bin/qemu-system-arm rmix,
+ /usr/bin/qemu-system-cris rmix,
+ /usr/bin/qemu-system-i386 rmix,
+ /usr/bin/qemu-system-lm32 rmix,
+ /usr/bin/qemu-system-m68k rmix,
+ /usr/bin/qemu-system-microblaze rmix,
+ /usr/bin/qemu-system-microblazeel rmix,
+ /usr/bin/qemu-system-mips rmix,
+ /usr/bin/qemu-system-mips64 rmix,
+ /usr/bin/qemu-system-mips64el rmix,
+ /usr/bin/qemu-system-mipsel rmix,
+ /usr/bin/qemu-system-moxie rmix,
+ /usr/bin/qemu-system-or32 rmix,
+ /usr/bin/qemu-system-ppc rmix,
+ /usr/bin/qemu-system-ppc64 rmix,
+ /usr/bin/qemu-system-ppc64le rmix,
+ /usr/bin/qemu-system-ppcemb rmix,
+ /usr/bin/qemu-system-s390x rmix,
+ /usr/bin/qemu-system-sh4 rmix,
+ /usr/bin/qemu-system-sh4eb rmix,
+ /usr/bin/qemu-system-sparc rmix,
+ /usr/bin/qemu-system-sparc64 rmix,
+ /usr/bin/qemu-system-tricore rmix,
+ /usr/bin/qemu-system-unicore32 rmix,
+ /usr/bin/qemu-system-x86_64 rmix,
+ /usr/bin/qemu-system-x86_64-spice rmix,
+ /usr/bin/qemu-system-xtensa rmix,
+ /usr/bin/qemu-system-xtensaeb rmix,
+ /usr/bin/qemu-aarch64 rmix,
+ /usr/bin/qemu-alpha rmix,
+ /usr/bin/qemu-arm rmix,
+ /usr/bin/qemu-armeb rmix,
+ /usr/bin/qemu-cris rmix,
+ /usr/bin/qemu-i386 rmix,
+ /usr/bin/qemu-m68k rmix,
+ /usr/bin/qemu-microblaze rmix,
+ /usr/bin/qemu-microblazeel rmix,
+ /usr/bin/qemu-mips rmix,
+ /usr/bin/qemu-mipsel rmix,
+ /usr/bin/qemu-mips64 rmix,
+ /usr/bin/qemu-mips64el rmix,
+ /usr/bin/qemu-mipsn32 rmix,
+ /usr/bin/qemu-mipsn32el rmix,
+ /usr/bin/qemu-or32 rmix,
+ /usr/bin/qemu-ppc rmix,
+ /usr/bin/qemu-ppc64 rmix,
+ /usr/bin/qemu-ppc64abi32 rmix,
+ /usr/bin/qemu-ppc64le rmix,
+ /usr/bin/qemu-s390x rmix,
+ /usr/bin/qemu-sh4 rmix,
+ /usr/bin/qemu-sh4eb rmix,
+ /usr/bin/qemu-sparc rmix,
+ /usr/bin/qemu-sparc64 rmix,
+ /usr/bin/qemu-sparc32plus rmix,
+ /usr/bin/qemu-sparc64 rmix,
+ /usr/bin/qemu-unicore32 rmix,
+ /usr/bin/qemu-x86_64 rmix,
+
+ # for save and resume
+ /bin/dash rmix,
+ /bin/dd rmix,
+ /bin/cat rmix,
+ /etc/pki/CA/ r,
+ /etc/pki/CA/* r,
+ /etc/pki/libvirt/ r,
+ /etc/pki/libvirt/** r,
+
+ # kvm.powerpc executes this
+ /bin/uname rmix,
+
+ # for rbd
+ /etc/ceph/ceph.conf r,
+
+ # for qemu-block-extra
+ /usr/lib/@{multiarch}/qemu/*.so rm,
+
+ # for access to hugepages
+ owner "/run/hugepages/kvm/libvirt/qemu/**" rw,
+ owner "/dev/hugepages/libvirt/qemu/**" rw,
+
+ # for usb access
+ /dev/bus/usb/ r,
+ /etc/udev/udev.conf r,
+ /sys/bus/ r,
+ /sys/class/ r,
+
+ signal (receive) peer=/usr/sbin/libvirtd,
+ ptrace (tracedby) peer=/usr/sbin/libvirtd,
+
+ # for ppc device-tree access
+ @{PROC}/device-tree/ r,
+ @{PROC}/device-tree/** r,
+ /sys/firmware/devicetree/** r,
+
+ # allow access to charm-specific ceph config (see lp#1403648)
+ /var/lib/charm/*/ceph.conf r,
+ # silence spurious denials (see lp#1403648)
+ deny /tmp/{,**} r,
+ deny /var/tmp/{,**} r,
+
+ # silence refusals to open lttng files (see lp#1432644)
+ deny /dev/shm/lttng-ust-wait-* r,
+ deny /run/shm/lttng-ust-wait-* r,
+
+ # allow serial console backed by pts chardev (LP: #1342083)
+ /usr/lib/pt_chown ix,
+ owner @{PROC}/0-9*/fd/ r,
+
+ # for vfio (LP: 1678322)
+ /dev/vfio/vfio rw,
+
+ /usr/{lib,libexec}/qemu-bridge-helper Cx -> qemu_bridge_helper,
+ # child profile for bridge helper process
+ profile qemu_bridge_helper {
+ #include <abstractions/base>
+
+ capability setuid,
+ capability setgid,
+ capability setpcap,
+ capability net_admin,
+
+ # for 9p
+ capability fsetid,
+ capability fowner,
+
+ network inet stream,
+
+ /dev/net/tun rw,
+ /etc/qemu/** r,
+ owner @{PROC}/*/status r,
+
+ /usr/{lib,libexec}/qemu-bridge-helper rmix,
+ }
diff --git a/kubernetes/files/kube-addons/virtlet/apparmor/libvirtd b/kubernetes/files/kube-addons/virtlet/apparmor/libvirtd
new file mode 100644
index 0000000..ae8ae88
--- /dev/null
+++ b/kubernetes/files/kube-addons/virtlet/apparmor/libvirtd
@@ -0,0 +1,72 @@
+# Profile is based on the upstream libvirt profile
+#include <tunables/global>
+
+profile libvirtd flags=(attach_disconnected) {
+ #include <abstractions/base>
+ #include <abstractions/dbus>
+
+ capability kill,
+ capability net_admin,
+ capability net_raw,
+ capability setgid,
+ capability sys_admin,
+ capability sys_module,
+ capability sys_ptrace,
+ capability sys_nice,
+ capability sys_chroot,
+ capability setuid,
+ capability dac_override,
+ capability dac_read_search,
+ capability fowner,
+ capability chown,
+ capability setpcap,
+ capability mknod,
+ capability fsetid,
+ capability ipc_lock,
+ capability audit_write,
+
+ # Needed for vfio
+ capability sys_resource,
+
+ network inet stream,
+ network inet dgram,
+ network inet6 stream,
+ network inet6 dgram,
+ network packet dgram,
+ network netlink,
+
+ dbus bus=system,
+ signal,
+ ptrace,
+ unix,
+
+ allow mount,
+ allow umount,
+
+ # for now, use a very lenient profile since we want to first focus on
+ # confining the guests
+ / r,
+ /** rwmkl,
+
+ /bin/* PUx,
+ /sbin/* PUx,
+ /usr/bin/* PUx,
+ /usr/sbin/* PUx,
+ /lib/udev/scsi_id PUx,
+ /usr/lib/xen-common/bin/xen-toolstack PUx,
+ /usr/lib/xen-*/bin/pygrub PUx,
+ /usr/lib/xen-*/bin/libxl-save-helper PUx,
+
+ # Required by nwfilter_ebiptables_driver.c:ebiptablesWriteToTempFile() to
+ # write and run an ebtables script.
+ /var/lib/libvirt/virtd* ixr,
+
+ /etc/libvirt/hooks/** rmix,
+ /etc/xen/scripts/** rmix,
+ /usr/lib/libvirt/* PUxr,
+ /usr/local/sbin/libvirtd rix,
+
+ /sys/kernel/security/apparmor/profiles r,
+
+ /vmwrapper rix,
+}
diff --git a/kubernetes/files/kube-addons/virtlet/apparmor/virtlet b/kubernetes/files/kube-addons/virtlet/apparmor/virtlet
new file mode 100644
index 0000000..2844319
--- /dev/null
+++ b/kubernetes/files/kube-addons/virtlet/apparmor/virtlet
@@ -0,0 +1,73 @@
+#include <tunables/global>
+
+profile virtlet flags=(attach_disconnected) {
+ #include <abstractions/base>
+ #include <abstractions/libvirt-qemu>
+ #include <abstractions/nameservice>
+
+ allow mount,
+ allow umount,
+ allow ptrace (read,trace) peer="unconfined",
+ capability net_admin,
+ capability net_raw,
+ capability sys_admin,
+ capability sys_chroot,
+ capability sys_ptrace,
+ network inet raw,
+ network inet6 raw,
+
+ / r,
+ /bin/sleep ix,
+ /etc/ethertypes r,
+ /etc/cni/net.d/ r,
+ /etc/cni/net.d/* r,
+ /etc/kubernetes/kubelet.kubeconfig r,
+ /etc/kubernetes/ssl/* r,
+ /etc/virtlet/images/ r,
+ /etc/virtlet/images/** r,
+ /{usr/,}bin/genisoimage rix,
+ /{usr/,}bin/socat rix,
+ /{usr/,}bin/ip rix,
+ /{usr/,}bin/nsenter rix,
+ /{usr/,}bin/qemu-img rix,
+ /{usr/,}sbin/ebtables rix,
+ /{usr/,}sbin/brctl rix,
+ /opt/cni/bin/calico* rix,
+ /opt/cni/bin/genie rix,
+ /usr{/local,}/bin/virtlet mrix,
+ /usr{/local,}/lib/lib{virt,guest}*.so* rm,
+ /var/lib/etcd/*.pem r,
+ /var/lib/calico/nodename r,
+ /var/lib/docker/overlay2/** r,
+ /var/lib/libvirt/virtd* ixr,
+ /var/lib/libvirt/*.sock rw,
+ /var/lib/virtlet/** rwk,
+ /var/lib/kubelet/pods/** rw,
+ /var/log/pods/** rw,
+ /{var/,}tmp/{,**} rw,
+
+ @{PROC}/@{pid}/net/psched r,
+ @{PROC}/@{pid}/net/ipv6_route r,
+ @{PROC}/@{pid}/status r,
+ @{PROC}/@{pid}/environ r,
+ @{PROC}/sys/kernel/hostname r,
+ @{PROC}/sys/net/core/somaxconn r,
+ @{PROC}/sys/net/ipv4/conf/cali*/* w,
+ @{PROC}/sys/net/ipv4/neigh/cali*/* w,
+ @{PROC}/sys/net/ipv4/ip_forward w,
+
+ /run/libvirt/libvirt-sock rw,
+ /run/virtlet.sock rw,
+ /run/virtlet-diag.sock rw,
+ /run/netns/ rw,
+ /run/netns/* rw,
+
+ /sys/class/net/ r,
+ /sys/devices/pci*/*/*/ r,
+ /sys/devices/pci*/*/*/* r,
+ /sys/devices/virtual/net/br*/bridge/ageing_time rw,
+ /sys/bus/pci/devices/ r,
+ /sys/bus/pci/devices/*/driver/unbind w,
+
+ /start.sh r,
+}
diff --git a/kubernetes/files/kube-addons/virtlet/apparmor/vms b/kubernetes/files/kube-addons/virtlet/apparmor/vms
new file mode 100644
index 0000000..7893e72
--- /dev/null
+++ b/kubernetes/files/kube-addons/virtlet/apparmor/vms
@@ -0,0 +1,19 @@
+#include <tunables/global>
+
+profile vms flags=(attach_disconnected) {
+ #include <abstractions/libvirt-qemu>
+
+ ptrace trace peer=@{profile_name},
+ ptrace readby,
+ ptrace tracedby,
+
+ /{usr/,}bin/sleep rix,
+ /{usr/,}bin/cut rix,
+ /{var/,}tmp/{,**} r,
+
+ /var/lib/virtlet/vms.procfile w,
+ /vms.sh rix,
+
+ @{PROC}/@{pid}/stat r,
+
+}
diff --git a/kubernetes/files/kube-addons/virtlet/virtlet-ds.yml b/kubernetes/files/kube-addons/virtlet/virtlet-ds.yml
index 35cd05f..51657a8 100644
--- a/kubernetes/files/kube-addons/virtlet/virtlet-ds.yml
+++ b/kubernetes/files/kube-addons/virtlet/virtlet-ds.yml
@@ -16,6 +16,12 @@
runtime: virtlet
template:
metadata:
+{%- if common.addons.virtlet.get('use_apparmor', False) %}
+ annotations:
+ container.apparmor.security.beta.kubernetes.io/libvirt: localhost/libvirtd
+ container.apparmor.security.beta.kubernetes.io/vms: localhost/vms
+ container.apparmor.security.beta.kubernetes.io/virtlet: localhost/virtlet
+{%- endif %}
creationTimestamp: null
labels:
runtime: virtlet
diff --git a/kubernetes/pool/init.sls b/kubernetes/pool/init.sls
index 117ca47..d5acd23 100644
--- a/kubernetes/pool/init.sls
+++ b/kubernetes/pool/init.sls
@@ -1,4 +1,5 @@
{%- from "kubernetes/map.jinja" import pool with context %}
+{%- from "kubernetes/map.jinja" import common with context -%}
include:
{%- if pool.network.get('calico', {}).get('enabled', False) %}
- kubernetes.pool.calico
@@ -19,3 +20,6 @@
{%- if pool.get('kube_proxy', {}).get('enabled', True) %}
- kubernetes.pool.kube-proxy
{%- endif %}
+{%- if common.addons.get('virtlet', {}).get('use_apparmor') and not pillar.get('kubernetes', {}).get('master', False) %}
+- kubernetes.pool.virtlet-apparmor
+{%- endif %}
diff --git a/kubernetes/pool/virtlet-apparmor.sls b/kubernetes/pool/virtlet-apparmor.sls
new file mode 100644
index 0000000..7dfabb9
--- /dev/null
+++ b/kubernetes/pool/virtlet-apparmor.sls
@@ -0,0 +1,30 @@
+{%- from "kubernetes/map.jinja" import common with context -%}
+{%- if common.addons.get('virtlet', {}).get('enabled') and common.addons.get('virtlet', {}).get('use_apparmor', False) %}
+
+{%- set apparmor_profiles = ['libvirtd', 'virtlet', 'vms'] %}
+
+{%- for profile in apparmor_profiles %}
+
+/etc/apparmor.d/{{ profile }}:
+ file.managed:
+ - source: salt://kubernetes/files/kube-addons/virtlet/apparmor/{{ profile }}
+ - user: root
+ - group: root
+ - mode: '0644'
+
+{%- endfor %}
+
+/etc/apparmor.d/abstractions/libvirt-qemu:
+ file.managed:
+ - source: salt://kubernetes/files/kube-addons/virtlet/apparmor/libvirt-qemu
+ - user: root
+ - group: root
+ - mode: '0644'
+
+apparmor:
+ service.running:
+ - watch:
+ - file: /etc/apparmor.d/*
+ - file: /etc/apparmor.d/abstractions/libvirt-qemu
+
+{% endif %}