Enable Virtlet Addon for Kubernetes
Enable virtlet addon for kubernetes deployment.
Change-Id: Ifd0cb2e01d46d21689ce7afb247b2614271731ac
Initiative: PROD-10135
Signed-off-by: Sergii Golovatiuk <sgolovatiuk@mirantis.com>
diff --git a/README.rst b/README.rst
index c162851..bf93fdb 100644
--- a/README.rst
+++ b/README.rst
@@ -61,6 +61,22 @@
calico_policy:
enabled: true
+Enable virtlet addon
+
+.. code-block:: yaml
+
+ parameters:
+ kubernetes:
+ master:
+ addons:
+ virtlet:
+ enabled: true
+ namespace: kube-system
+ hosts:
+ - cmp01
+ - cmp02
+ image: mirantis/virtlet:latest
+
Enable netchecker addon
.. code-block:: yaml
diff --git a/kubernetes/_common.sls b/kubernetes/_common.sls
index 113e67d..5171517 100644
--- a/kubernetes/_common.sls
+++ b/kubernetes/_common.sls
@@ -1,4 +1,4 @@
-{% from "kubernetes/map.jinja" import common with context %}
+{%- from "kubernetes/map.jinja" import common with context %}
kubernetes_pkgs:
pkg.installed:
@@ -55,13 +55,75 @@
- onlyif: /bin/false
{%- endif %}
+/tmp/criproxy:
+ file.directory:
+ - user: root
+ - group: root
+
+copy-criproxy-bin:
+ cmd.run:
+ - name: docker run --rm -v /tmp/criproxy/:/tmp/criproxy/ --entrypoint cp mirantis/virtlet -vr /criproxy /tmp/criproxy
+ - require:
+ - file: /tmp/criproxy
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+
+/usr/bin/criproxy:
+ file.managed:
+ - source: /tmp/criproxy/criproxy
+ - mode: 750
+ - makedirs: true
+ - user: root
+ - group: root
+ - require:
+ - cmd: copy-criproxy-bin
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+
+/etc/criproxy:
+ file.directory:
+ - user: root
+ - group: root
+ - mode: 0750
+
+/etc/criproxy/kubelet.conf:
+ file.managed:
+ - source: salt://kubernetes/files/virtlet/kubelet.conf
+ - template: jinja
+ - user: root
+ - group: root
+ - mode: 640
+
+/etc/systemd/system/criproxy.service:
+ file.managed:
+ - source: salt://kubernetes/files/systemd/criproxy.service
+ - template: jinja
+ - user: root
+ - group: root
+ - mode: 755
+
+criproxy_service:
+ service.running:
+ - name: criproxy
+ - enable: True
+ - watch:
+ - file: /etc/systemd/system/criproxy.service
+ - file: /etc/criproxy/kubelet.conf
+ - file: /etc/criproxy
+ - file: /usr/bin/criproxy
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+
/etc/systemd/system/kubelet.service:
file.managed:
- - source: salt://kubernetes/files/systemd/kubelet.service
- - template: jinja
- - user: root
- - group: root
- - mode: 644
+ - source: salt://kubernetes/files/systemd/kubelet.service
+ - template: jinja
+ - user: root
+ - group: root
+ - mode: 644
/etc/kubernetes/config:
file.absent
diff --git a/kubernetes/files/kube-addons/kube-network-manager/kube-network-manager-deploy.yml b/kubernetes/files/kube-addons/kube-network-manager/kube-network-manager-deploy.yml
index 571db7e..2e6a622 100644
--- a/kubernetes/files/kube-addons/kube-network-manager/kube-network-manager-deploy.yml
+++ b/kubernetes/files/kube-addons/kube-network-manager/kube-network-manager-deploy.yml
@@ -20,7 +20,7 @@
imagePullPolicy: Always
args: ["--config-file", "/etc/kube-manager/contrail.conf", "--alsologtostderr"]
volumeMounts:
- - name: kube-manager
+ - name: kube-network-manager
mountPath: /etc/kube-manager/
volumes:
- name: kube-network-manager
diff --git a/kubernetes/files/kube-addons/virtlet/virtlet-ds.yml b/kubernetes/files/kube-addons/virtlet/virtlet-ds.yml
new file mode 100644
index 0000000..511580c
--- /dev/null
+++ b/kubernetes/files/kube-addons/virtlet/virtlet-ds.yml
@@ -0,0 +1,239 @@
+{%- from "kubernetes/map.jinja" import master with context %}
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+ name: virtlet
+ namespace: {{ master.addons.virtlet.namespace }}
+spec:
+ template:
+ metadata:
+ name: virtlet
+ labels:
+ runtime: virtlet
+ spec:
+ hostNetwork: true
+ {%- if master.network.engine != "opencontrail" %}
+ dnsPolicy: ClusterFirstWithHostNet
+ {%- endif %}
+ # hostPID is true to (1) enable VMs to survive virtlet container restart
+ # (to be checked) and (2) to enable the use of nsenter in init container
+ hostPID: true
+ # bootstrap procedure needs to create a configmap in kube-system namespace
+ serviceAccountName: virtlet
+
+ # only run Virtlet pods on the nodes with extraRuntime=virtlet label
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: extraRuntime
+ operator: In
+ values:
+ - virtlet
+
+ initContainers:
+ # The init container first copies virtlet's flexvolume driver
+ # to the default kubelet plugin dir to have it in the proper place by the
+ # time kubelet is restarted by CRI proxy bootstrap procedure.
+ # After that it checks if there's already saved kubelet config
+ # and considers that CRI proxy bootstrap is already done if it exists.
+ # If it doesn't, it drops criproxy binary into /opt/criproxy/bin
+ # if it's not already there and then starts criproxy installation.
+ # The possibility to put criproxy binary in advance into
+ # /opt/criproxy/bin may be helpful for the purpose of
+ # debugging criproxy
+ # At the end it ensures that /var/lib/libvirt/images exists on node.
+ - name: prepare-node
+ image: {{ master.addons.virtlet.image }}
+ imagePullPolicy: IfNotPresent
+ command:
+ - /prepare-node.sh
+ volumeMounts:
+ - name: k8s-flexvolume-plugins-dir
+ mountPath: /kubelet-volume-plugins
+ - name: criproxybin
+ mountPath: /opt/criproxy/bin
+ - name: run
+ mountPath: /run
+ - name: dockersock
+ mountPath: /var/run/docker.sock
+ - name: criproxyconf
+ mountPath: /etc/criproxy
+ - name: log
+ mountPath: /hostlog
+ # for ensuring that /var/lib/libvirt/images exists on node
+ - name: var-lib
+ mountPath: /host-var-lib
+
+ containers:
+ - name: virtlet
+ image: {{ master.addons.virtlet.image }}
+ # In case we inject local virtlet image we want to use it not officially available one
+ imagePullPolicy: IfNotPresent
+ volumeMounts:
+ - mountPath: /sys/fs/cgroup
+ name: cgroup
+ - mountPath: /lib/modules
+ name: modules
+ readOnly: true
+ - mountPath: /boot
+ name: boot
+ readOnly: true
+ - mountPath: /run
+ name: run
+ - mountPath: /var/lib/virtlet
+ name: virtlet
+ - mountPath: /var/lib/libvirt
+ name: libvirt
+ - mountPath: /etc/cni
+ name: cniconf
+ - mountPath: /opt/cni/bin
+ name: cnibin
+ - mountPath: /var/lib/cni
+ name: cnidata
+ - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+ name: k8s-flexvolume-plugins-dir
+ # below `:shared` is unofficial way to pass this option docker
+ # which then will allow virtlet to see what kubelet mounts in
+ # underlaying directories, after virtlet container is created
+ - mountPath: /var/lib/kubelet/pods:shared
+ name: k8s-pods-dir
+ - name: vms-log
+ mountPath: /var/log/vms
+ {%- if master.network.engine == "opencontrail" %}
+ - name: contrail-log
+ mountPath: /var/log/contrail
+ - name: contrail-data
+ mountPath: /var/lib/contrail
+ {%- endif %}
+ securityContext:
+ privileged: true
+ env:
+ - name: VIRTLET_LOGLEVEL
+ value: "3"
+ - name: VIRTLET_DOWNLOAD_PROTOCOL
+ value: "https"
+ # Uncomment the following to disable KVM:
+ # - name: VIRTLET_DISABLE_KVM
+ # value: "y"
+ # Uncomment the following to redirect VM logs to file /var/log/vms/<sandboxId>/<containerId>_<attemptIdx>.log:
+ - name: VIRTLET_VM_LOG_LOCATION
+ value: "/var/log/vms"
+ - name: virtlet-log
+ image: {{ master.addons.virtlet.image }}
+ imagePullPolicy: IfNotPresent
+ command:
+ - /virtlet_log
+ volumeMounts:
+ - name: vms-log
+ mountPath: /virtlet-log
+ - name: pods-log
+ mountPath: /kubernetes-log
+ env:
+ - name: VIRTLET_VM_LOGS
+ value: "/virtlet-log"
+ - name: KUBERNETES_POD_LOGS
+ value: "/kubernetes-log"
+ volumes:
+ - hostPath:
+ path: /sys/fs/cgroup
+ name: cgroup
+ - hostPath:
+ path: /lib/modules
+ name: modules
+ - hostPath:
+ path: /boot
+ name: boot
+ - hostPath:
+ path: /run
+ name: run
+ # TODO: don't hardcode docker socket location here
+ # This will require CRI proxy installation to run
+ # in host mount namespace.
+ - hostPath:
+ path: /var/run/docker.sock
+ name: dockersock
+ - hostPath:
+ path: /var/lib/virtlet
+ name: virtlet
+ - hostPath:
+ path: /var/lib/libvirt
+ name: libvirt
+ - hostPath:
+ path: /etc/cni
+ name: cniconf
+ - hostPath:
+ path: /opt/cni/bin
+ name: cnibin
+ - hostPath:
+ path: /var/lib/cni
+ name: cnidata
+ - hostPath:
+ path: /opt/criproxy/bin
+ name: criproxybin
+ - hostPath:
+ path: /etc/criproxy
+ name: criproxyconf
+ - hostPath:
+ path: /var/log
+ name: log
+ - hostPath:
+ path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+ name: k8s-flexvolume-plugins-dir
+ - hostPath:
+ path: /var/lib/kubelet/pods
+ name: k8s-pods-dir
+ - hostPath:
+ path: /var/lib
+ name: var-lib
+ - hostPath:
+ path: /var/log/virtlet/vms
+ name: vms-log
+ - hostPath:
+ path: /var/log/pods
+ name: pods-log
+ {%- if master.network.engine == "opencontrail" %}
+ - hostPath:
+ path: /var/log/contrail
+ name: contrail-log
+ - hostPath:
+ path: /var/lib/contrail
+ name: contrail-data
+ - hostPath:
+ path: /virtlet
+ name: virtlet-bin
+ {%- endif %}
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+ name: virtlet
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: virtlet
+subjects:
+- kind: ServiceAccount
+ name: virtlet
+ namespace: {{ master.addons.virtlet.namespace }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: virtlet
+ namespace: {{ master.addons.virtlet.namespace }}
+rules:
+ - apiGroups:
+ - ""
+ resources:
+ - configmaps
+ verbs:
+ - create
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: virtlet
+ namespace: {{ master.addons.virtlet.namespace }}
diff --git a/kubernetes/files/kubelet/default.pool b/kubernetes/files/kubelet/default.pool
index e191738..df49d4c 100644
--- a/kubernetes/files/kubelet/default.pool
+++ b/kubernetes/files/kubelet/default.pool
@@ -16,6 +16,10 @@
--network-plugin-dir=/etc/cni/net.d \
{%- endif %}
--file-check-frequency={{ pool.kubelet.frequency }} \
+--container-runtime=remote \
+--container-runtime-endpoint=/var/run/criproxy.sock \
+--image-service-endpoint=/var/run/criproxy.sock \
+--enable-controller-attach-detach=false \
{%- for key, value in pool.get('kubelet', {}).get('daemon_opts', {}).iteritems() %}
--{{ key }}="{{ value }}" \
{% endfor %}
diff --git a/kubernetes/files/systemd/criproxy.service b/kubernetes/files/systemd/criproxy.service
new file mode 100644
index 0000000..6d91cb2
--- /dev/null
+++ b/kubernetes/files/systemd/criproxy.service
@@ -0,0 +1,14 @@
+[Unit]
+Description=CRI Proxy
+
+[Service]
+ExecStart=/usr/bin/criproxy -alsologtostderr \
+ -connect docker,virtlet:/var/run/virtlet.sock \
+ -kubeletcfg /etc/criproxy/kubelet.conf \
+ -listen /var/run/criproxy.sock
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=kubelet.service
diff --git a/kubernetes/files/virtlet/kubelet.conf b/kubernetes/files/virtlet/kubelet.conf
new file mode 100644
index 0000000..e8ff7d5
--- /dev/null
+++ b/kubernetes/files/virtlet/kubelet.conf
@@ -0,0 +1,142 @@
+{
+ "address": "0.0.0.0",
+ "allowPrivileged": true,
+ "authentication": {
+ "anonymous": {
+ "enabled": true
+ },
+ "webhook": {
+ "cacheTTL": "2m0s",
+ "enabled": false
+ },
+ "x509": {
+ "clientCAFile": ""
+ }
+ },
+ "authorization": {
+ "mode": "AlwaysAllow",
+ "webhook": {
+ "cacheAuthorizedTTL": "5m0s",
+ "cacheUnauthorizedTTL": "30s"
+ }
+ },
+ "babysitDaemons": false,
+ "cAdvisorPort": 4194,
+ "certDirectory": "/var/run/kubernetes",
+ "cgroupDriver": "cgroupfs",
+ "cgroupRoot": "",
+ "cgroupsPerQOS": true,
+ "cloudConfigFile": "",
+ "cloudProvider": "auto-detect",
+ "clusterDNS": [
+ "10.254.0.10"
+ ],
+ "clusterDomain": "cluster.local",
+ "cniBinDir": "/opt/cni/bin",
+ "cniConfDir": "",
+ "containerRuntime": "docker",
+ "containerized": false,
+ "contentType": "application/vnd.kubernetes.protobuf",
+ "cpuCFSQuota": true,
+ "dockerEndpoint": "unix:///var/run/docker.sock",
+ "dockerExecHandlerName": "native",
+ "enableCRI": true,
+ "enableContentionProfiling": false,
+ "enableControllerAttachDetach": true,
+ "enableCustomMetrics": false,
+ "enableDebuggingHandlers": true,
+ "enableServer": true,
+ "enforceNodeAllocatable": [
+ "pods"
+ ],
+ "eventBurst": 10,
+ "eventRecordQPS": 5,
+ "evictionHard": "memory.available<100Mi",
+ "evictionMaxPodGracePeriod": 0,
+ "evictionMinimumReclaim": "",
+ "evictionPressureTransitionPeriod": "5m0s",
+ "evictionSoft": "",
+ "evictionSoftGracePeriod": "",
+ "exitOnLockContention": false,
+ "experimentalKernelMemcgNotification": false,
+ "experimentalQOSReserved": {},
+ "featureGates": "DynamicKubeletConfig=true",
+ "fileCheckFrequency": "5s",
+ "hairpinMode": "promiscuous-bridge",
+ "healthzBindAddress": "127.0.0.1",
+ "healthzPort": 10248,
+ "hostIPCSources": [
+ "*"
+ ],
+ "hostNetworkSources": [
+ "*"
+ ],
+ "hostPIDSources": [
+ "*"
+ ],
+ "hostnameOverride": "",
+ "httpCheckFrequency": "20s",
+ "imageGCHighThresholdPercent": 90,
+ "imageGCLowThresholdPercent": 80,
+ "imageMinimumGCAge": "2m0s",
+ "imagePullProgressDeadline": "1m0s",
+ "iptablesDropBit": 15,
+ "iptablesMasqueradeBit": 14,
+ "kubeAPIBurst": 10,
+ "kubeAPIQPS": 5,
+ "kubeReserved": {},
+ "kubeletCgroups": "",
+ "lockFilePath": "",
+ "lowDiskSpaceThresholdMB": 256,
+ "makeIPTablesUtilChains": true,
+ "manifestURL": "",
+ "manifestURLHeader": "",
+ "masterServiceNamespace": "default",
+ "maxContainerCount": -1,
+ "maxOpenFiles": 1000000,
+ "maxPerPodContainerCount": 1,
+ "maxPods": 110,
+ "minimumGCAge": "0s",
+ "networkPluginDir": "/etc/cni/net.d",
+ "networkPluginMTU": 0,
+ "networkPluginName": "cni",
+ "nodeIP": "",
+ "nodeLabels": {
+ "node-role.kubernetes.io/node": "true"
+ },
+ "nodeStatusUpdateFrequency": "10s",
+ "nonMasqueradeCIDR": "10.0.0.0/8",
+ "oomScoreAdj": -999,
+ "outOfDiskTransitionFrequency": "5m0s",
+ "podCIDR": "",
+ "podInfraContainerImage": "gcr.io/google_containers/pause-amd64:3.0",
+ "podManifestPath": "/etc/kubernetes/manifests",
+ "podsPerCore": 0,
+ "port": 10250,
+ "protectKernelDefaults": false,
+ "readOnlyPort": 10255,
+ "registerNode": true,
+ "registerSchedulable": true,
+ "registerWithTaints": [],
+ "registryBurst": 10,
+ "registryPullQPS": 5,
+ "remoteImageEndpoint": "",
+ "remoteRuntimeEndpoint": "",
+ "resolvConf": "/etc/resolv.conf",
+ "rktAPIEndpoint": "localhost:15441",
+ "rktPath": "",
+ "rktStage1Image": "",
+ "rootDirectory": "/var/lib/kubelet",
+ "runtimeCgroups": "",
+ "runtimeRequestTimeout": "2m0s",
+ "seccompProfileRoot": "/var/lib/kubelet/seccomp",
+ "serializeImagePulls": true,
+ "streamingConnectionIdleTimeout": "4h0m0s",
+ "syncFrequency": "1m0s",
+ "systemCgroups": "",
+ "systemReserved": {},
+ "tlsCertFile": "",
+ "tlsPrivateKeyFile": "",
+ "volumePluginDir": "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
+ "volumeStatsAggPeriod": "1m0s"
+}
diff --git a/kubernetes/master/kube-addons.sls b/kubernetes/master/kube-addons.sls
index 2fd3675..ea857f5 100644
--- a/kubernetes/master/kube-addons.sls
+++ b/kubernetes/master/kube-addons.sls
@@ -8,7 +8,7 @@
- group: root
- mode: 0755
-{%- if master.addons.get('kube_network_manager', {}).get('enabled', False) and master.network.engine == "opencontrail" %}
+{%- if master.network.engine == "opencontrail" %}
/etc/kubernetes/addons/kube_network_manager/kube-network-manager-configmap.yml:
file.managed:
- source: salt://kubernetes/files/kube-addons/kube-network-manager/kube-network-manager-configmap.yml
@@ -27,6 +27,17 @@
{% endif %}
+{%- if master.addons.get('virtlet', {}).get('enabled') %}
+/etc/kubernetes/addons/virtlet/virtlet-ds.yml:
+ file.managed:
+ - source: salt://kubernetes/files/kube-addons/virtlet/virtlet-ds.yml
+ - template: jinja
+ - group: root
+ - dir_mode: 755
+ - makedirs: True
+
+{% endif %}
+
{%- if master.addons.get('calico_policy', {}).get('enabled', False) and master.network.engine == "calico" %}
/etc/kubernetes/addons/calico_policy/calico-policy-controller.yml:
file.managed:
diff --git a/kubernetes/master/opencontrail.sls b/kubernetes/master/opencontrail.sls
index d5d4cb3..246b216 100644
--- a/kubernetes/master/opencontrail.sls
+++ b/kubernetes/master/opencontrail.sls
@@ -1,17 +1,5 @@
{%- from "kubernetes/map.jinja" import master with context %}
{%- if master.enabled %}
-
-linklocal_apiserver:
- contrail.linklocal_service_present:
- - name: kube_apiserver
- - lls_ip: "{{ master.apiserver.internal_address }}"
- - lls_port: 443
- - ipf_addresses: "{{ master.apiserver.address }}"
- - ipf_port: 443
- {%- if grains.get('noservices') %}
- - onlyif: /bin/false
- {%- endif %}
-
{%- if master.network.get('version', '3.0') != '3.0' %}
opencontrail_kube_manager_package:
diff --git a/kubernetes/master/setup.sls b/kubernetes/master/setup.sls
index 5d22277..b7d3806 100644
--- a/kubernetes/master/setup.sls
+++ b/kubernetes/master/setup.sls
@@ -63,5 +63,18 @@
{%- endif %}
+{%- if master.addons.get('virtlet', {}).get('enabled') %}
+{% for host in master.addons.virtlet.hosts %}
+
+label_virtlet_{{ host }}:
+ cmd.run:
+ - name: kubectl label --overwrite node {{ host }} extraRuntime=virtlet
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+
+{% endfor %}
+
+{%- endif %}
{%- endif %}
diff --git a/metadata/service/master/cluster.yml b/metadata/service/master/cluster.yml
index 1e38d03..c25acf4 100644
--- a/metadata/service/master/cluster.yml
+++ b/metadata/service/master/cluster.yml
@@ -72,7 +72,7 @@
virtlet:
enabled: False
namespace: kube-system
- image: Mirantis/virtlet:latest
+ image: mirantis/virtlet:latest
token:
admin: ${_param:kubernetes_admin_token}
kubelet: ${_param:kubernetes_kubelet_token}
diff --git a/metadata/service/master/single.yml b/metadata/service/master/single.yml
index 1a9cbf5..0418488 100644
--- a/metadata/service/master/single.yml
+++ b/metadata/service/master/single.yml
@@ -53,6 +53,10 @@
enabled: False
image: calico/kube-policy-controller:v0.5.4
namespace: kube-system
+ virtlet:
+ enabled: False
+ namespace: kube-system
+ image: mirantis/virtlet:latest
token:
admin: ${_param:kubernetes_admin_token}
kubelet: ${_param:kubernetes_kubelet_token}
diff --git a/tests/pillar/master_cluster.sls b/tests/pillar/master_cluster.sls
index 6f41bc4..a2c98fd 100644
--- a/tests/pillar/master_cluster.sls
+++ b/tests/pillar/master_cluster.sls
@@ -34,6 +34,13 @@
enabled: true
namespace: kube-system
image: image
+ virtlet:
+ enabled: true
+ namespace: kube-system
+ hosts:
+ - cmp01
+ - cmp02
+ image: mirantis/virtlet:latest
admin:
password: password
username: admin
diff --git a/tests/pillar/master_contrail4_0.sls b/tests/pillar/master_contrail4_0.sls
index 7ff0e50..b687840 100644
--- a/tests/pillar/master_contrail4_0.sls
+++ b/tests/pillar/master_contrail4_0.sls
@@ -31,6 +31,9 @@
server_image: image
agent_image: image
agent_probeurls: "http://ipinfo.io"
+ kube_network_manager:
+ enabled: true
+ namespace: kube-system
admin:
password: password
username: admin
@@ -53,8 +56,9 @@
kubelet:
allow_privileged: true
network:
- version: 4.0
engine: opencontrail
+ version: 4.0
+ private_ip_range: 10.150.0.0/16
config:
api:
host: 127.0.0.1