Merge "Fix of deploy virtlet."
diff --git a/kubernetes/_common.sls b/kubernetes/_common.sls
index b489517..283956e 100644
--- a/kubernetes/_common.sls
+++ b/kubernetes/_common.sls
@@ -89,6 +89,28 @@
- require:
- file: /usr/bin/criproxy
+{%- if not pillar.kubernetes.pool is defined %}
+
+/etc/default/dockershim:
+ file.managed:
+ - source: salt://kubernetes/files/dockershim/default.master
+ - template: jinja
+ - user: root
+ - group: root
+ - mode: 644
+
+{%- else %}
+
+/etc/default/dockershim:
+ file.managed:
+ - source: salt://kubernetes/files/dockershim/default.pool
+ - template: jinja
+ - user: root
+ - group: root
+ - mode: 644
+
+{%- endif %}
+
/etc/criproxy:
file.directory:
- user: root
@@ -123,7 +145,7 @@
- name: dockershim
- enable: True
- watch:
- - file: /etc/systemd/system/dockershim.service
+ - file: /etc/default/dockershim
- file: /usr/bin/dockershim
{%- if grains.get('noservices') %}
- onlyif: /bin/false
diff --git a/kubernetes/files/dockershim/default.master b/kubernetes/files/dockershim/default.master
new file mode 100644
index 0000000..1e30e39
--- /dev/null
+++ b/kubernetes/files/dockershim/default.master
@@ -0,0 +1,27 @@
+{%- from "kubernetes/map.jinja" import common with context -%}
+{%- from "kubernetes/map.jinja" import master with context -%}
+
+# test_args has to be kept at the end, so they'll overwrite any prior configuration
+DAEMON_ARGS="\
+--pod-manifest-path=/etc/kubernetes/manifests \
+--address={{ master.kubelet.address }} \
+--allow-privileged={{ master.kubelet.allow_privileged }} \
+--cluster_dns={{ common.addons.dns.server }} \
+--cluster_domain={{ common.addons.dns.domain|replace('_', '-') }} \
+--cni-bin-dir={{ master.apiserver.get('cni_bin_dir', '/opt/cni/bin') }} \
+--hostname-override={{ master.host.name }} \
+--v={{ master.get('verbosity', 2) }} \
+--node-labels=node-role.kubernetes.io/master=true \
+{%- if master.get('unschedulable', 'false') %}
+--register-with-taints=node-role.kubernetes.io/master=:NoSchedule \
+{%- endif %}
+{%- if common.get('cloudprovider', {}).get('enabled') %}
+--cloud-provider={{ common.cloudprovider.provider }} \
+{%- if common.get('cloudprovider', {}).get('provider') == 'openstack' %}
+--cloud-config=/etc/kubernetes/cloud-config.conf \
+{%- endif %}
+{%- endif %}
+{%- for key, value in master.get('kubelet', {}).get('daemon_opts', {}).iteritems() %}
+--{{ key }}={{ value }} \
+{%- endfor %}
+"
diff --git a/kubernetes/files/dockershim/default.pool b/kubernetes/files/dockershim/default.pool
new file mode 100644
index 0000000..cd717c4
--- /dev/null
+++ b/kubernetes/files/dockershim/default.pool
@@ -0,0 +1,42 @@
+{%- from "kubernetes/map.jinja" import common with context -%}
+{%- from "kubernetes/map.jinja" import pool with context -%}
+
+# test_args has to be kept at the end, so they'll overwrite any prior configuration
+DAEMON_ARGS="\
+--require-kubeconfig \
+--kubeconfig=/etc/kubernetes/kubelet.kubeconfig \
+--pod-manifest-path=/etc/kubernetes/manifests \
+--address={{ pool.kubelet.address }} \
+--allow-privileged={{ pool.kubelet.allow_privileged }} \
+--cluster_dns={{ common.addons.dns.server }} \
+--cluster_domain={{ common.addons.dns.domain|replace('_', '-') }} \
+--cni-bin-dir={{ pool.apiserver.get('cni_bin_dir', '/opt/cni/bin') }} \
+--hostname-override={{ pool.host.name }} \
+--v={{ pool.get('verbosity', 2) }} \
+{%- if pillar.kubernetes.master is defined %}
+--node-labels=node-role.kubernetes.io/master=true \
+{%- if pillar.kubernetes.get('master', {}).get('unschedulable', 'false') %}
+--register-with-taints=node-role.kubernetes.io/master=:NoSchedule \
+{%- endif -%}
+{%- else %}
+--node-labels=node-role.kubernetes.io/node=true \
+{%- endif %}
+{%- if pool.network.engine in ['calico', 'opencontrail'] %}
+--network-plugin=cni \
+--network-plugin-dir=/etc/cni/net.d \
+{%- endif %}
+--file-check-frequency={{ pool.kubelet.frequency }} \
+{%- if common.get('cloudprovider', {}).get('enabled') %}
+--cloud-provider={{ common.cloudprovider.provider }} \
+{%- if common.get('cloudprovider', {}).get('provider') == 'openstack' %}
+--cloud-config=/etc/kubernetes/cloud-config.conf \
+{%- endif %}
+{%- endif %}
+{%- if common.addons.get('virtlet', {}).get('enabled') %}
+--container-runtime={{ pool.get('container-runtime', 'remote') }} \
+--enable-controller-attach-detach={{ pool.get('enable-controller-attach-detach', 'false') }} \
+{%- endif %}
+{%- for key, value in pool.get('kubelet', {}).get('daemon_opts', {}).iteritems() %}
+--{{ key }}={{ value }} \
+{%- endfor %}
+"
diff --git a/kubernetes/files/kube-addons/virtlet/virtlet-ds.yml b/kubernetes/files/kube-addons/virtlet/virtlet-ds.yml
index b1c75a3..eaced34 100644
--- a/kubernetes/files/kube-addons/virtlet/virtlet-ds.yml
+++ b/kubernetes/files/kube-addons/virtlet/virtlet-ds.yml
@@ -69,12 +69,16 @@
# for ensuring that /var/lib/libvirt/images exists on node
- name: var-lib
mountPath: /host-var-lib
+ securityContext:
+ privileged: true
containers:
- - name: virtlet
+ - name: libvirt
image: {{ common.addons.virtlet.image }}
# In case we inject local virtlet image we want to use it not officially available one
imagePullPolicy: IfNotPresent
+ command:
+ - /libvirt.sh
volumeMounts:
- mountPath: /sys/fs/cgroup
name: cgroup
@@ -90,10 +94,52 @@
name: virtlet
- mountPath: /var/lib/libvirt
name: libvirt
+ - mountPath: /var/run/libvirt
+ name: libvirt-sockets
+ # the log dir is needed here because otherwise libvirt will produce errors
+ # like this:
+ # Unable to pre-create chardev file '/var/log/vms/afd75bbb-8e97-11e7-9561-02420ac00002/cirros-vm_0.log': No such file or directory
+ - name: vms-log
+ mountPath: /var/log/vms
+ - name: dev
+ mountPath: /dev
+ securityContext:
+ privileged: true
+ env:
+ - name: VIRTLET_DISABLE_KVM
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: disable_kvm
+ optional: true
+ - name: virtlet
+ image: {{ common.addons.virtlet.image }}
+ # In case we inject local virtlet image we want to use it not officially available one
+ imagePullPolicy: IfNotPresent
+ volumeMounts:
+ - mountPath: /var/lib/etcd
+ name: etcd
+ - mountPath: /run
+ name: run
+ # /boot and /lib/modules are required by supermin
+ - mountPath: /lib/modules
+ name: modules
+ readOnly: true
+ - mountPath: /boot
+ name: boot
+ readOnly: true
+ - mountPath: /var/lib/virtlet
+ name: virtlet
+ - mountPath: /var/lib/libvirt
+ name: libvirt
+ - mountPath: /etc/kubernetes
+ name: kubernetes
- mountPath: /etc/cni
name: cniconf
- mountPath: /opt/cni/bin
name: cnibin
+ - mountPath: /var/run/libvirt
+ name: libvirt-sockets
- mountPath: /var/lib/cni
name: cnidata
- mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
@@ -111,35 +157,75 @@
- name: contrail-data
mountPath: /var/lib/contrail
{%- endif %}
+ - mountPath: /etc/virtlet/images
+ name: image-name-translations
+ - name: pods-log
+ mountPath: /kubernetes-log
securityContext:
privileged: true
env:
- - name: VIRTLET_LOGLEVEL
- value: "3"
+ - name: VIRTLET_DISABLE_KVM
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: disable_kvm
+ optional: true
- name: VIRTLET_DOWNLOAD_PROTOCOL
- value: "https"
- # Uncomment the following to disable KVM:
- # - name: VIRTLET_DISABLE_KVM
- # value: "y"
- # Uncomment the following to redirect VM logs to file /var/log/vms/<sandboxId>/<containerId>_<attemptIdx>.log:
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: download_protocol
+ optional: true
+ - name: VIRTLET_LOGLEVEL
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: loglevel
+ optional: true
+ - name: VIRTLET_CALICO_SUBNET
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: calico-subnet
+ optional: true
+ - name: IMAGE_REGEXP_TRANSLATION
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: image_regexp_translation
+ optional: true
+ - name: IMAGE_TRANSLATIONS_DIR
+ value: /etc/virtlet/images
+ - name: KUBERNETES_POD_LOGS
+ value: "/kubernetes-log"
+ # TODO: should we rename it?
- name: VIRTLET_VM_LOG_LOCATION
- value: "/var/log/vms"
- - name: virtlet-log
+ value: "1"
+ - name: vms
image: {{ common.addons.virtlet.image }}
imagePullPolicy: IfNotPresent
command:
- - /virtlet_log
+ - /vms.sh
volumeMounts:
+ - mountPath: /var/lib/virtlet
+ name: virtlet
+ - mountPath: /var/lib/libvirt
+ name: libvirt
- name: vms-log
- mountPath: /virtlet-log
- - name: pods-log
- mountPath: /kubernetes-log
- env:
- - name: VIRTLET_VM_LOGS
- value: "/virtlet-log"
- - name: KUBERNETES_POD_LOGS
- value: "/kubernetes-log"
+ mountPath: /var/log/vms
+ - name: dev
+ mountPath: /dev
volumes:
+ # /dev is needed for host raw device access
+ - hostPath:
+ path: /dev
+ name: dev
+ - hostPath:
+ path: /etc/kubernetes
+ name: kubernetes
+ - hostPath:
+ path: /var/lib/etcd
+ name: etcd
- hostPath:
path: /sys/fs/cgroup
name: cgroup
@@ -195,8 +281,14 @@
path: /var/log/virtlet/vms
name: vms-log
- hostPath:
+ path: /var/run/libvirt
+ name: libvirt-sockets
+ - hostPath:
path: /var/log/pods
name: pods-log
+ - configMap:
+ name: virtlet-image-translations
+ name: image-name-translations
{%- if master.network.engine == "opencontrail" %}
- hostPath:
path: /var/log/contrail
@@ -210,34 +302,74 @@
{%- endif %}
---
apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+ name: configmap-reader
+ namespace: {{ common.addons.virtlet.namespace }}
+ labels:
+ addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - configmaps
+ verbs:
+ - get
+ - list
+ - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
- name: virtlet
+ name: kubelet-node-binding
labels:
addonmanager.kubernetes.io/mode: Reconcile
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
- name: virtlet
+ name: configmap-reader
subjects:
-- kind: ServiceAccount
- name: virtlet
+- apiGroup: rbac.authorization.k8s.io
+ kind: Group
+ name: system:nodes
namespace: {{ common.addons.virtlet.namespace }}
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
- name: virtlet
+ name: virtlet-crd
namespace: {{ common.addons.virtlet.namespace }}
labels:
addonmanager.kubernetes.io/mode: Reconcile
rules:
- apiGroups:
- - ""
+ - "apiextensions.k8s.io"
resources:
- - configmaps
+ - customresourcedefinitions
verbs:
- create
+ - apiGroups:
+ - "virtlet.k8s"
+ resources:
+ - virtletimagemappings
+ verbs:
+ - list
+ - get
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+ name: virtlet-crd
+ labels:
+ addonmanager.kubernetes.io/mode: Reconcile
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: virtlet-crd
+subjects:
+- kind: ServiceAccount
+ name: virtlet
+ namespace: {{ common.addons.virtlet.namespace }}
---
apiVersion: v1
kind: ServiceAccount
@@ -246,3 +378,16 @@
namespace: {{ common.addons.virtlet.namespace }}
labels:
addonmanager.kubernetes.io/mode: Reconcile
+---
+apiVersion: v1
+data:
+ images.yml: |
+ translations:
+ - name: cirros
+ url: https://github.com/mirantis/virtlet/releases/download/v0.8.2/cirros.img
+kind: ConfigMap
+metadata:
+ name: virtlet-image-translations
+ namespace: {{ common.addons.virtlet.namespace }}
+ labels:
+ addonmanager.kubernetes.io/mode: Reconcile
\ No newline at end of file
diff --git a/kubernetes/files/kubelet/default.pool b/kubernetes/files/kubelet/default.pool
index ea2f928..8207a7d 100644
--- a/kubernetes/files/kubelet/default.pool
+++ b/kubernetes/files/kubelet/default.pool
@@ -1,5 +1,6 @@
{%- from "kubernetes/map.jinja" import common with context -%}
{%- from "kubernetes/map.jinja" import pool with context -%}
+{%- from "kubernetes/map.jinja" import version %}
# test_args has to be kept at the end, so they'll overwrite any prior configuration
DAEMON_ARGS="\
@@ -34,8 +35,13 @@
{%- endif %}
{%- if common.addons.get('virtlet', {}).get('enabled') %}
--container-runtime={{ pool.get('container-runtime', 'remote') }} \
+{%- if version|float < 1.7 %}
--container-runtime-endpoint={{ pool.get('container-runtime-endpoint', '/var/run/criproxy.sock') }} \
--image-service-endpoint={{ pool.get('image-service-endpoint', '/var/run/criproxy.sock') }} \
+{%- else %}
+--container-runtime-endpoint={{ pool.get('container-runtime-endpoint', 'unix:///var/run/criproxy.sock') }} \
+--image-service-endpoint={{ pool.get('image-service-endpoint', 'unix:///var/run/criproxy.sock') }} \
+{%- endif %}
--enable-controller-attach-detach={{ pool.get('enable-controller-attach-detach', 'false') }} \
{%- endif %}
{%- for key, value in pool.get('kubelet', {}).get('daemon_opts', {}).iteritems() %}
diff --git a/kubernetes/files/systemd/criproxy.service b/kubernetes/files/systemd/criproxy.service
index fb6dcf5..1c0318c 100644
--- a/kubernetes/files/systemd/criproxy.service
+++ b/kubernetes/files/systemd/criproxy.service
@@ -1,13 +1,15 @@
[Unit]
Description=CRI Proxy
+After=dockershim.service
+Requires=dockershim.service
[Service]
+SyslogIdentifier=criproxy
+User=root
ExecStart=/usr/bin/criproxy -alsologtostderr \
-connect /var/run/dockershim.sock,virtlet:/var/run/virtlet.sock \
-listen /var/run/criproxy.sock
-Restart=always
-StartLimitInterval=0
-RestartSec=10
+Restart=on-failure
[Install]
WantedBy=kubelet.service
diff --git a/kubernetes/files/systemd/dockershim.service b/kubernetes/files/systemd/dockershim.service
index ec02645..eb2eb99 100644
--- a/kubernetes/files/systemd/dockershim.service
+++ b/kubernetes/files/systemd/dockershim.service
@@ -1,8 +1,15 @@
[Unit]
Description=dockershim for criproxy
+Documentation=https://github.com/kubernetes/kubernetes/tree/master/pkg/kubelet/dockershim
+After=network.target
+After=docker.service
+Requires=docker.service
[Service]
-EnvironmentFile=-/etc/default/kubelet
+SyslogIdentifier=dockershim
+EnvironmentFile=-/etc/kubernetes/config
+EnvironmentFile=-/etc/default/%p
+User=root
ExecStart=/usr/bin/dockershim \
$KUBE_LOGTOSTDERR \
$KUBE_LOG_LEVEL \
@@ -14,11 +21,7 @@
$DOCKER_ENDPOINT \
$CADVISOR_PORT \
$DAEMON_ARGS
-
-Restart=always
-StartLimitInterval=0
-RestartSec=10
+Restart=on-failure
[Install]
-RequiredBy=criproxy.service
-
+WantedBy=kubelet.service
\ No newline at end of file
diff --git a/kubernetes/map.jinja b/kubernetes/map.jinja
index daa7fe7..4382f88 100644
--- a/kubernetes/map.jinja
+++ b/kubernetes/map.jinja
@@ -1,15 +1,15 @@
-{% set _version = salt['cmd.shell']("(hyperkube --version 2> /dev/null || echo '0.0') | sed -e 's/-.*//g' -e 's/v//g' -e 's/Kubernetes //g' | awk -F'.' '{print $1 \".\" $2}'") %}
+{% set version = salt['cmd.shell']("(hyperkube --version 2> /dev/null || echo '0.0') | sed -e 's/-.*//g' -e 's/v//g' -e 's/Kubernetes //g' | awk -F'.' '{print $1 \".\" $2}'") %}
{% set common = salt['grains.filter_by']({
'Debian': {
'pkgs': ['curl', 'git', 'apt-transport-https', 'python-apt', 'socat', 'openssl'],
'services': [],
- 'version': _version|float,
+ 'version': version|float,
},
'RedHat': {
'pkgs': ['curl', 'git', 'socat', 'python', 'openssl'],
'services': [],
- 'version': _version|float,
+ 'version': version|float,
},
}, merge=salt['pillar.get']('kubernetes:common')) %}