Merge "Adjust virtlet-ds"
diff --git a/kubernetes/files/kube-addons/virtlet/virtlet-ds.yml b/kubernetes/files/kube-addons/virtlet/virtlet-ds.yml
index 1e75fb1..7a2372e 100644
--- a/kubernetes/files/kube-addons/virtlet/virtlet-ds.yml
+++ b/kubernetes/files/kube-addons/virtlet/virtlet-ds.yml
@@ -18,6 +18,15 @@
runtime: virtlet
name: virtlet
spec:
+ hostNetwork: true
+ dnsPolicy: ClusterFirstWithHostNet
+ # hostPID is true to (1) enable VMs to survive virtlet container restart
+ # (to be checked) and (2) to enable the use of nsenter in init container
+ hostPID: true
+ # bootstrap procedure needs to create a configmap in kube-system namespace
+ serviceAccountName: virtlet
+
+ # only run Virtlet pods on the nodes with extraRuntime=virtlet label
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
@@ -27,25 +36,97 @@
operator: In
values:
- virtlet
- containers:
- - command:
- - /libvirt.sh
+ initContainers:
+ # The init container copies virtlet's flexvolume driver
+ # to the default kubelet plugin dir and ensures that
+ # the directories needed by libvirt & virtlet exist on the host
+ - name: prepare-node
+ image: {{ common.addons.virtlet.image }}
+ imagePullPolicy: IfNotPresent
+ command:
+ - /prepare-node.sh
+ volumeMounts:
+ - name: k8s-flexvolume-plugins-dir
+ mountPath: /kubelet-volume-plugins
+ - name: run
+ mountPropagation: Bidirectional
+ mountPath: /run
+ - name: dockersock
+ mountPath: /var/run/docker.sock
+ - name: log
+ mountPath: /hostlog
+ # for ensuring that /var/lib/libvirt/images exists on node
+ - name: var-lib
+ mountPath: /host-var-lib
+ - name: dev
+ mountPath: /dev
+ - mountPath: /var/lib/virtlet
+ name: virtlet
+ securityContext:
+ privileged: true
env:
- - name: VIRTLET_SRIOV_SUPPORT
+ - name: KUBE_NODE_NAME
valueFrom:
- configMapKeyRef:
- key: sriov_support
- name: virtlet-config
- optional: true
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
- name: VIRTLET_DISABLE_KVM
valueFrom:
configMapKeyRef:
- key: disable_kvm
name: virtlet-config
+ key: disable_kvm
optional: true
+ - name: VIRTLET_SRIOV_SUPPORT
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: sriov_support
+ optional: true
+ - name: VIRTLET_DOWNLOAD_PROTOCOL
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: download_protocol
+ optional: true
+ - name: VIRTLET_LOGLEVEL
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: loglevel
+ optional: true
+ - name: VIRTLET_CALICO_SUBNET
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: calico-subnet
+ optional: true
+ - name: IMAGE_REGEXP_TRANSLATION
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: image_regexp_translation
+ optional: true
+ - name: VIRTLET_RAW_DEVICES
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: raw_devices
+ optional: true
+ - name: VIRTLET_DISABLE_LOGGING
+ valueFrom:
+ configMapKeyRef:
+ name: virtlet-config
+ key: disable_logging
+ optional: true
+ - name: VIRTLET_IMAGE_TRANSLATIONS_DIR
+ value: /etc/virtlet/images
+
+ containers:
+ - name: libvirt
image: {{ common.addons.virtlet.image }}
imagePullPolicy: IfNotPresent
- name: libvirt
+ command:
+ - /libvirt.sh
readinessProbe:
exec:
command:
@@ -65,87 +146,40 @@
name: boot
readOnly: true
- mountPath: /run
- mountPropagation: Bidirectional
name: run
+ mountPropagation: Bidirectional
- mountPath: /var/lib/virtlet
name: virtlet
- mountPath: /var/lib/libvirt
name: libvirt
- mountPath: /var/run/libvirt
name: libvirt-sockets
- - mountPath: /var/log/vms
- name: vms-log
- - mountPath: /var/log/libvirt
- name: libvirt-log
- - mountPath: /dev
- name: dev
- - env:
- - name: VIRTLET_DISABLE_KVM
- valueFrom:
- configMapKeyRef:
- key: disable_kvm
- name: virtlet-config
- optional: true
- - name: VIRTLET_DOWNLOAD_PROTOCOL
- valueFrom:
- configMapKeyRef:
- key: download_protocol
- name: virtlet-config
- optional: true
- - name: VIRTLET_LOGLEVEL
- valueFrom:
- configMapKeyRef:
- key: loglevel
- name: virtlet-config
- optional: true
- - name: VIRTLET_CALICO_SUBNET
- valueFrom:
- configMapKeyRef:
- key: calico-subnet
- name: virtlet-config
- optional: true
- - name: IMAGE_REGEXP_TRANSLATION
- valueFrom:
- configMapKeyRef:
- key: image_regexp_translation
- name: virtlet-config
- optional: true
- - name: VIRTLET_DISABLE_LOGGING
- valueFrom:
- configMapKeyRef:
- key: disable_logging
- name: virtlet-config
- optional: true
- - name: VIRTLET_SRIOV_SUPPORT
- valueFrom:
- configMapKeyRef:
- key: sriov_support
- name: virtlet-config
- optional: true
- - name: VIRTLET_RAW_DEVICES
- valueFrom:
- configMapKeyRef:
- key: raw_devices
- name: virtlet-config
- optional: true
- - name: IMAGE_TRANSLATIONS_DIR
- value: /etc/virtlet/images
- image: {{ common.addons.virtlet.image }}
- imagePullPolicy: IfNotPresent
- name: virtlet
+ # the log dir is needed here because otherwise libvirt will produce errors
+ # like this:
+ # Unable to pre-create chardev file '/var/log/vms/afd75bbb-8e97-11e7-9561-02420ac00002/cirros-vm_0.log': No such file or directory
+ - name: vms-log
+ mountPath: /var/log/vms
+ - name: libvirt-log
+ mountPath: /var/log/libvirt
+ - name: dev
+ mountPath: /dev
+ securityContext:
+ privileged: true
readinessProbe:
exec:
command:
- /bin/sh
- -c
- - socat - UNIX:/run/virtlet.sock </dev/null
- resources: {}
- securityContext:
- privileged: true
+ - socat - UNIX:/var/run/libvirt/libvirt-sock-ro </dev/null
+ - name: virtlet
+ image: {{ common.addons.virtlet.image }}
+ # In case we inject local virtlet image we want to use it not officially available one
+ imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /run
- mountPropagation: Bidirectional
name: run
+ mountPropagation: Bidirectional
+ # /boot and /lib/modules are required by supermin
- mountPath: /lib/modules
name: modules
readOnly: true
@@ -161,66 +195,40 @@
- mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
name: k8s-flexvolume-plugins-dir
- mountPath: /var/lib/kubelet/pods
- mountPropagation: Bidirectional
name: k8s-pods-dir
- - mountPath: /var/log/vms
- name: vms-log
+ mountPropagation: Bidirectional
+ - name: vms-log
+ mountPath: /var/log/vms
- mountPath: /etc/virtlet/images
name: image-name-translations
- - mountPath: /var/log/pods
- name: pods-log
- - command:
- - /vms.sh
+ - name: pods-log
+ mountPath: /var/log/pods
+ securityContext:
+ privileged: true
+ readinessProbe:
+ exec:
+ command:
+ - /bin/sh
+ - -c
+ - socat - UNIX:/run/virtlet.sock </dev/null
+ - name: vms
image: {{ common.addons.virtlet.image }}
imagePullPolicy: IfNotPresent
- name: vms
- resources: {}
+ command:
+ - /vms.sh
volumeMounts:
- mountPath: /var/lib/virtlet
name: virtlet
- mountPath: /var/lib/libvirt
name: libvirt
- - mountPath: /var/log/vms
- name: vms-log
- - mountPath: /dev
- name: dev
- - mountPath: /lib/modules
- name: modules
- dnsPolicy: ClusterFirstWithHostNet
- hostNetwork: true
- hostPID: true
- initContainers:
- - command:
- - /prepare-node.sh
- env:
- - name: VIRTLET_DISABLE_KVM
- valueFrom:
- configMapKeyRef:
- key: disable_kvm
- name: virtlet-config
- optional: true
- image: {{ common.addons.virtlet.image }}
- imagePullPolicy: IfNotPresent
- name: prepare-node
- resources: {}
- securityContext:
- privileged: true
- volumeMounts:
- - mountPath: /kubelet-volume-plugins
- name: k8s-flexvolume-plugins-dir
- - mountPath: /run
- mountPropagation: Bidirectional
- name: run
- - mountPath: /var/run/docker.sock
- name: dockersock
- - mountPath: /hostlog
- name: log
- - mountPath: /host-var-lib
- name: var-lib
- - mountPath: /dev
- name: dev
- serviceAccountName: virtlet
+ - name: vms-log
+ mountPath: /var/log/vms
+ - name: dev
+ mountPath: /dev
+ - name: modules
+ mountPath: /lib/modules
volumes:
+ # /dev is needed for host raw device access
- hostPath:
path: /dev
name: dev
@@ -236,6 +244,9 @@
- hostPath:
path: /run
name: run
+ # TODO: don't hardcode docker socket location here
+ # This will require CRI proxy installation to run
+ # in host mount namespace.
- hostPath:
path: /var/run/docker.sock
name: dockersock
@@ -275,13 +286,6 @@
- configMap:
name: virtlet-image-translations
name: image-name-translations
- updateStrategy: {}
-status:
- currentNumberScheduled: 0
- desiredNumberScheduled: 0
- numberMisscheduled: 0
- numberReady: 0
-
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
@@ -298,7 +302,6 @@
- kind: ServiceAccount
name: virtlet
namespace: {{ common.addons.virtlet.namespace }}
-
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
@@ -309,13 +312,14 @@
labels:
addonmanager.kubernetes.io/mode: Reconcile
rules:
-- apiGroups:
- - ""
- resources:
- - configmaps
- verbs:
- - create
-
+ - apiGroups:
+ - ""
+ resources:
+ - configmaps
+ - nodes
+ verbs:
+ - create
+ - get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
@@ -333,7 +337,6 @@
- get
- list
- watch
-
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
@@ -350,7 +353,6 @@
- secrets
verbs:
- get
-
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
@@ -367,7 +369,6 @@
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:nodes
-
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
@@ -384,7 +385,6 @@
- kind: ServiceAccount
name: virtlet
namespace: {{ common.addons.virtlet.namespace }}
-
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
@@ -404,10 +404,10 @@
- virtlet.k8s
resources:
- virtletimagemappings
+ - virtletconfigmappings
verbs:
- list
- get
-
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
@@ -424,7 +424,6 @@
- kind: ServiceAccount
name: virtlet
namespace: {{ common.addons.virtlet.namespace }}
-
---
apiVersion: v1
kind: ServiceAccount
@@ -435,6 +434,109 @@
labels:
addonmanager.kubernetes.io/mode: Reconcile
---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ creationTimestamp: null
+ name: virtletimagemappings.virtlet.k8s
+ namespace: {{ common.addons.virtlet.namespace }}
+ labels:
+ virtlet.cloud: ""
+ addonmanager.kubernetes.io/mode: Reconcile
+spec:
+ group: virtlet.k8s
+ names:
+ kind: VirtletImageMapping
+ plural: virtletimagemappings
+ shortNames:
+ - vim
+ singular: virtletimagemapping
+ scope: Namespaced
+ version: v1
+status:
+ acceptedNames:
+ kind: ""
+ plural: ""
+ conditions: null
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ creationTimestamp: null
+ name: virtletconfigmappings.virtlet.k8s
+ namespace: {{ common.addons.virtlet.namespace }}
+ labels:
+ virtlet.cloud: ""
+ addonmanager.kubernetes.io/mode: Reconcile
+spec:
+ group: virtlet.k8s
+ names:
+ kind: VirtletConfigMapping
+ plural: virtletconfigmappings
+ shortNames:
+ - vcm
+ singular: virtletconfigmapping
+ scope: Namespaced
+ validation:
+ openAPIV3Schema:
+ properties:
+ spec:
+ properties:
+ config:
+ properties:
+ calicoSubnetSize:
+ maximum: 32
+ minimum: 0
+ type: integer
+ cniConfigDir:
+ type: string
+ cniPluginDir:
+ type: string
+ criSocketPath:
+ type: string
+ databasePath:
+ type: string
+ disableKVM:
+ type: boolean
+ disableLogging:
+ type: boolean
+ downloadProtocol:
+ pattern: ^https?$
+ type: string
+ enableRegexpImageTranslation:
+ type: boolean
+ enableSriov:
+ type: boolean
+ fdServerSocketPath:
+ type: string
+ type: string
+ imageDir:
+ type: string
+ imageTranslationConfigsDir:
+ type: string
+ libvirtURI:
+ type: string
+ logLevel:
+ maximum: 2147483647
+ minimum: 0
+ type: integer
+ rawDevices:
+ type: string
+ skipImageTranslation:
+ type: boolean
+ nodeName:
+ type: string
+ nodeSelector:
+ type: object
+ priority:
+ type: integer
+ version: v1
+status:
+ acceptedNames:
+ kind: ""
+ plural: ""
+ conditions: null
+---
apiVersion: v1
data:
images.yml: |