Merge "Add support of Helm client."
diff --git a/.kitchen.yml b/.kitchen.yml
index 74272ff..5a065c2 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -51,16 +51,10 @@
sudo: true
docker_images:
- - &xenial-20163 <%=ENV['IMAGE_XENIAL_20163'] || 'docker-dev-local.docker.mirantis.net/epcim/salt/saltstack-ubuntu-xenial-salt-2016.3/salt:2018_11_19'%>
- &xenial-20177 <%=ENV['IMAGE_XENIAL_20177'] || 'docker-dev-local.docker.mirantis.net/epcim/salt/saltstack-ubuntu-xenial-salt-2017.7/salt:2018_11_19'%>
- &xenial-stable <%=ENV['IMAGE_XENIAL_STABLE'] || 'docker-dev-local.docker.mirantis.net/epcim/salt/saltstack-ubuntu-xenial-salt-stable/salt:2018_11_19'%>
platforms:
- - name: xenial-2016.3
- driver_config:
- image: *xenial-20163
- platform: ubuntu
-
- name: xenial-2017.7
driver_config:
image: *xenial-20177
diff --git a/README.rst b/README.rst
index 263d1a3..3cf026d 100644
--- a/README.rst
+++ b/README.rst
@@ -1185,6 +1185,29 @@
test:
kind: User
+Manage client (kubectl resources)
+Assign labels to nodes:
+
+.. code-block:: yaml
+
+ kubernetes:
+ client:
+ enabled: true
+ apiserver:
+ insecure_address: 0.0.0.0
+ insecure_port: 8080
+ resources:
+ enabled: true
+ label:
+ test:
+ value: enabled
+ status: 'present'
+ node:
+ - cmp1
+ - cmp2
+ enabled: true
+ key: mylabel
+
More Information
================
diff --git a/kubernetes/_common.sls b/kubernetes/_common.sls
index b74a76a..3d59fbf 100644
--- a/kubernetes/_common.sls
+++ b/kubernetes/_common.sls
@@ -36,6 +36,7 @@
/etc/containerd/config.toml:
file.managed:
- source: salt://kubernetes/files/containerd/config.toml
+ - makedirs: True
- template: jinja
- user: root
- group: root
@@ -134,6 +135,12 @@
- group: root
- mode: 0750
+/var/log/criproxy:
+ file.directory:
+ - user: root
+ - group: root
+ - mode: 0750
+
/etc/criproxy/node.conf:
file.managed:
- user: root
@@ -192,6 +199,7 @@
file.absent
{%- if common.get('cloudprovider', {}).get('enabled') and common.get('cloudprovider', {}).get('provider') == 'openstack' %}
+{%- set cloudconfig_type = 'external' %}
/etc/kubernetes/cloud-config:
file.managed:
- source: salt://kubernetes/files/cloudprovider/cloud-config-openstack.conf
@@ -199,6 +207,21 @@
- user: root
- group: root
- mode: 600
+ - defaults:
+ cloudconfig_type: {{ cloudconfig_type }}
+
+{%- if pillar.kubernetes.master is defined %}
+{%- set cloudconfig_type = 'intree' %}
+/etc/kubernetes/cloud-config.intree:
+ file.managed:
+ - source: salt://kubernetes/files/cloudprovider/cloud-config-openstack.conf
+ - template: jinja
+ - user: root
+ - group: root
+ - mode: 600
+ - defaults:
+ cloudconfig_type: {{ cloudconfig_type }}
+{% endif %}
{% endif %}
@@ -294,7 +317,7 @@
- options: xzf
{%- endif %}
- archive_format: tar
- - if_missing: /opt/cni/bin/host-local
+ - overwrite: true
{%- endif %}
{%- if common.addons.get('helm', {'enabled': False}).enabled %}
diff --git a/kubernetes/client.sls b/kubernetes/client.sls
new file mode 100644
index 0000000..9c1c383
--- /dev/null
+++ b/kubernetes/client.sls
@@ -0,0 +1,33 @@
+{%- from "kubernetes/map.jinja" import client with context -%}
+{%- if client.enabled %}
+ {%- if client.get('resources', {}).get('enabled') %}
+
+ {%- for name,label in client.resources.get('label', {}).iteritems() %}
+
+ {%- if label.enabled %}
+ {%- if label.get('status', 'present') == 'present' %}
+ {%- for node in label.node %}
+# TODO(vsaienko) switch to kubernetes. salt module once kubernets-client python is packages and
+# awailable for installation.
+{{ name }}_{{ node }}:
+ k8s.label_present:
+ - name: {{ label.key }}
+ - value: {{ label.value }}
+ - node: {{ node }}
+ # TODO(vsaienko): move to profiles
+ - apiserver: http://{{ client.apiserver.insecure_address }}:{{ client.apiserver.insecure_port }}
+ {%- endfor %}
+
+ {%- elif label.get('status', 'present') == 'absent' %}
+ {%- for node in label.node %}
+{{ name }}_{{ node }}:
+ k8s.label_absent:
+ - name: {{ label.key }}
+ - node: {{ node }}
+ - apiserver: http://{{ client.apiserver.insecure_address }}:{{ client.apiserver.insecure_port }}
+ {%- endfor %} # endfor label.node.iteritems
+ {%- endif %} # endif label.present
+ {%- endif %} # endif label.enabled
+ {%- endfor %} # endfor client.resources.label
+ {%- endif %} # endif client.resources.enabled
+{%- endif %} # endif client.enabled
diff --git a/kubernetes/control/endpoint.sls b/kubernetes/control/endpoint.sls
new file mode 100644
index 0000000..962855e
--- /dev/null
+++ b/kubernetes/control/endpoint.sls
@@ -0,0 +1,60 @@
+{% from "kubernetes/map.jinja" import control with context %}
+include:
+ - kubernetes.control
+
+{%- for endpoint_name, endpoint in control.endpoints.items() %}
+ {%- if endpoint.get('service_enabled', false) %}
+
+/srv/kubernetes/services/{{ endpoint.cluster }}/{{ endpoint.service }}-svc.yml:
+ file.managed:
+ - source: salt://kubernetes/files/svc.yml
+ - user: root
+ - group: root
+ - template: jinja
+ - makedirs: true
+ - require:
+ - file: /srv/kubernetes
+ - defaults:
+ service: {{ endpoint|yaml }}
+
+ {%- if endpoint.get('create', false) %}
+ {%- set service_name = endpoint.service + '-' + endpoint.role if endpoint.role is defined else endpoint.service %}
+kubernetes_service_create_{{ endpoint.service }}:
+ cmd.wait:
+ - name: kubectl apply -f /srv/kubernetes/services/{{ endpoint.cluster }}/{{ endpoint.service }}-svc.yml
+ - unless: kubectl get service -o=custom-columns=NAME:.metadata.name --namespace {{ endpoint.namespace }} | grep -xq {{ endpoint.service }}
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+ - watch:
+ - file: /srv/kubernetes/services/{{ endpoint.cluster }}/{{ endpoint.service }}-svc.yml
+ {%- endif %}
+
+ {%- endif %}
+
+/srv/kubernetes/endpoints/{{ endpoint.cluster }}/{{ endpoint_name }}.yml:
+ file.managed:
+ - source: salt://kubernetes/files/endpoint.yml
+ - user: root
+ - group: root
+ - template: jinja
+ - makedirs: true
+ - require:
+ - file: /srv/kubernetes
+ - defaults:
+ endpoint: {{ endpoint|yaml }}
+ endpoint_name: {{ endpoint_name }}
+
+ {%- if endpoint.get('create', false) %}
+kubernetes_endpoint_create_{{ endpoint_name }}:
+ cmd.wait:
+ - name: kubectl apply -f /srv/kubernetes/endpoints/{{ endpoint.cluster }}/{{ endpoint_name }}.yml
+ - unless: kubectl get endpoint -o=custom-columns=NAME:.metadata.name --namespace {{ endpoint.namespace }} | grep -xq {{ endpoint_name }}
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+ - watch:
+ - file: /srv/kubernetes/endpoints/{{ endpoint.cluster }}/{{ endpoint_name }}.yml
+ {%- endif %}
+
+{%- endfor %}
diff --git a/kubernetes/control/ingress.sls b/kubernetes/control/ingress.sls
new file mode 100644
index 0000000..8a262dc
--- /dev/null
+++ b/kubernetes/control/ingress.sls
@@ -0,0 +1,34 @@
+{% from "kubernetes/map.jinja" import control with context %}
+include:
+ - kubernetes.control
+
+{%- for ingress_name, ingress in control.ingress.items() %}
+ {%- if ingress.get('enabled', false) %}
+
+/srv/kubernetes/ingress/{{ ingress.cluster }}/{{ ingress_name }}-ingress.yml:
+ file.managed:
+ - source: salt://kubernetes/files/ingress.yml
+ - user: root
+ - group: root
+ - template: jinja
+ - makedirs: true
+ - require:
+ - file: /srv/kubernetes
+ - defaults:
+ ingress: {{ ingress|yaml }}
+ ingress_name: {{ ingress_name }}
+
+ {%- if ingress.get('create', false) %}
+kubernetes_ingress_create_{{ ingress_name }}:
+ cmd.wait:
+ - name: kubectl apply -f /srv/kubernetes/ingress/{{ ingress.cluster }}/{{ ingress_name }}-ingress.yml
+ - unless: kubectl get ingress -o=custom-columns=NAME:.metadata.name --namespace {{ ingress.namespace }} | grep -xq {{ ingress_name }}
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+ - watch:
+ - file: /srv/kubernetes/ingress/{{ ingress.cluster }}/{{ ingress_name }}-ingress.yml
+ {%- endif %}
+
+ {%- endif %}
+{%- endfor %}
diff --git a/kubernetes/control/init.sls b/kubernetes/control/init.sls
index b8ea755..18d8b11 100644
--- a/kubernetes/control/init.sls
+++ b/kubernetes/control/init.sls
@@ -15,6 +15,12 @@
{%- if control.priorityclass is defined %}
- kubernetes.control.priorityclass
{%- endif %}
+ {%- if control.endpoints is defined %}
+ - kubernetes.control.endpoint
+ {%- endif %}
+ {%- if control.ingress is defined %}
+ - kubernetes.control.ingress
+ {%- endif %}
/srv/kubernetes:
file.directory:
diff --git a/kubernetes/control/service.sls b/kubernetes/control/service.sls
index e7e9330..4e94a56 100644
--- a/kubernetes/control/service.sls
+++ b/kubernetes/control/service.sls
@@ -3,7 +3,7 @@
- kubernetes.control
{%- for service_name, service in control.service.items() %}
- {%- if service.enabled %}
+ {%- if service.get('enabled', false) %}
/srv/kubernetes/services/{{ service.cluster }}/{{ service_name }}-svc.yml:
file.managed:
@@ -17,6 +17,19 @@
- defaults:
service: {{ service|yaml }}
+ {%- if service.get('create', false) %}
+ {%- set service_real_name = service.service + '-' + service.role if service.role is defined else service.service %}
+kubernetes_service_create_{{ service_name }}:
+ cmd.wait:
+ - name: kubectl apply -f /srv/kubernetes/services/{{ service.cluster }}/{{ service_name }}-svc.yml
+ - unless: kubectl get service -o=custom-columns=NAME:.metadata.name --namespace {{ service.namespace }} | grep -xq {{ service_real_name }}
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+ - watch:
+ - file: /srv/kubernetes/services/{{ service.cluster }}/{{ service_name }}-svc.yml
+ {%- endif %}
+
{%- endif %}
/srv/kubernetes/{{ service.kind|lower }}/{{ service_name }}-{{ service.kind }}.yml:
@@ -31,6 +44,19 @@
- defaults:
service: {{ service|yaml }}
+ {%- if service.get('create', false) %}
+ {%- set service_real_name = service.service + '-' + service.role if service.role is defined else service.service %}
+kubernetes_{{ service.kind|lower }}_create_{{ service_name }}:
+ cmd.wait:
+ - name: kubectl apply -f /srv/kubernetes/{{ service.kind|lower }}/{{ service_name }}-{{ service.kind }}.yml
+ - unless: kubectl get {{ service.kind|lower }} -o=custom-columns=NAME:.metadata.name --namespace {{ service.namespace }} | grep -xq {{ service_real_name }}
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+ - watch:
+ - file: /srv/kubernetes/{{ service.kind|lower }}/{{ service_name }}-{{ service.kind }}.yml
+ {%- endif %}
+
{%- endfor %}
{%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').items() %}
@@ -51,6 +77,19 @@
- defaults:
service: {{ service|yaml }}
+ {%- if service.get('create', false) %}
+ {%- set service_real_name = service.service + '-' + service.role if service.role is defined else service.service %}
+kubernetes_service_create_{{ service.service }}:
+ cmd.wait:
+ - name: kubectl apply -f /srv/kubernetes/services/{{ node_name }}-svc.yml
+ - unless: kubectl get service -o=custom-columns=NAME:.metadata.name --namespace {{ service.namespace }} | grep -xq {{ service_real_name }}
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+ - watch:
+ - file: /srv/kubernetes/services/{{ node_name }}-svc.yml
+ {%- endif %}
+
{%- endif %}
/srv/kubernetes/{{ service.kind|lower }}/{{ node_name }}-{{ service.kind }}.yml:
file.managed:
@@ -64,6 +103,19 @@
- defaults:
service: {{ service|yaml }}
+ {%- if service.get('create', false) %}
+ {%- set service_real_name = service.service + '-' + service.role if service.role is defined else service.service %}
+kubernetes_{{ service.kind|lower }}_create_{{ service_name }}:
+ cmd.wait:
+ - name: kubectl apply -f /srv/kubernetes/{{ service.kind|lower }}/{{ node_name }}-{{ service.kind }}.yml
+ - unless: kubectl get {{ service.kind|lower }} -o=custom-columns=NAME:.metadata.name --namespace {{ service.namespace }} | grep -xq {{ service_real_name }}
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+ - watch:
+ - file: /srv/kubernetes/{{ service.kind|lower }}/{{ node_name }}-{{ service.kind }}.yml
+ {%- endif %}
+
{%- endif %}
{%- endfor %}
diff --git a/kubernetes/files/cloudprovider/cloud-config-openstack.conf b/kubernetes/files/cloudprovider/cloud-config-openstack.conf
index 92a1cdd..83bdd8d 100644
--- a/kubernetes/files/cloudprovider/cloud-config-openstack.conf
+++ b/kubernetes/files/cloudprovider/cloud-config-openstack.conf
@@ -22,15 +22,15 @@
[LoadBalancer]
-{%- if common.cloudprovider.params.subnet_id is defined %}
use-octavia=true
+{%- if common.cloudprovider.params.subnet_id is defined %}
subnet-id={{ common.cloudprovider.params.subnet_id }}
{%- endif %}
{%- if common.cloudprovider.params.lb_method is defined %}
lb-method={{ common.cloudprovider.params.lb_method }}
{%- endif %}
-{%- if common.cloudprovider.params.floating_network_id is defined %}
-floating-network-id={{ common.cloudprovider.params.floating_network_id }}
+{%- if common.cloudprovider.params.floating_net_id is defined %}
+floating-network-id={{ common.cloudprovider.params.floating_net_id }}
{%- endif %}
{%- if common.cloudprovider.params.create_monitor is defined %}
create-monitor={{ common.cloudprovider.params.create_monitor }}
@@ -46,3 +46,17 @@
[BlockStorage]
ignore-volume-az=true
+
+
+{%- if pillar.kubernetes.master is defined and cloudconfig_type == "external" %}
+[Networking]
+{%- if common.cloudprovider.params.internal_net_name is defined %}
+internal-network-name={{ common.cloudprovider.params.internal_net_name }}
+{%- endif %}
+{%- if common.cloudprovider.params.public_net_name is defined %}
+public-network-name={{ common.cloudprovider.params.public_net_name }}
+{%- endif %}
+{%- if common.cloudprovider.params.ipv6_support_disabled is defined %}
+ipv6-support-disabled={{ common.cloudprovider.params.ipv6_support_disabled }}
+{%- endif %}
+{%- endif %}
diff --git a/kubernetes/files/endpoint.yml b/kubernetes/files/endpoint.yml
new file mode 100644
index 0000000..6109bf8
--- /dev/null
+++ b/kubernetes/files/endpoint.yml
@@ -0,0 +1,18 @@
+{% from "kubernetes/map.jinja" import control with context %}
+apiVersion: v1
+kind: Endpoints
+metadata:
+ name: {{ endpoint_name }}
+ {%- if endpoint.namespace is defined %}
+ namespace: {{ endpoint.namespace }}
+ {%- endif %}
+subsets:
+{%- for subset in endpoint.subsets %}
+ - addresses:
+ - ip: {{ subset.ip }}
+ ports:
+ - port: {{ subset.port.number }}
+ {%- if subset.port.name is defined %}
+ name: {{ subset.port.name }}
+ {%- endif %}
+{%- endfor %}
diff --git a/kubernetes/files/ingress.yml b/kubernetes/files/ingress.yml
new file mode 100644
index 0000000..5fb1cf0
--- /dev/null
+++ b/kubernetes/files/ingress.yml
@@ -0,0 +1,41 @@
+{% from "kubernetes/map.jinja" import control with context %}
+apiVersion: {{ ingress.apiVersion }}
+kind: Ingress
+metadata:
+ name: {{ ingress_name }}
+ namespace: {{ ingress.namespace }}
+ {%- if ingress.annotations is defined %}
+ annotations:
+ {%- for annotation in ingress.annotations %}
+ {{ annotation.name }}: "{{ annotation.value }}"
+ {%- endfor %}
+ {%- endif %}
+spec:
+ {%- if ingress.tls is defined %}
+ tls:
+ - hosts:
+ {%- for host in ingress.tls.hosts %}
+ - {{ host }}
+ {%- endfor %}
+ secretName: {{ ingress.tls.secret_name }}
+ {%- endif %}
+ {%- if ingress.rules is defined %}
+ rules:
+ {%- for host in ingress.rules.hosts %}
+ - {%- if host.name is defined %}
+ host: {{ host.name }}
+ {%- endif %}
+ http:
+ paths:
+ {%- for path, backend in host.paths.items() %}
+ - path: {{ path }}
+ backend:
+ serviceName: {{ backend.service }}
+ servicePort: {{ backend.port }}
+ {%- endfor %}
+ {%- endfor %}
+ {%- elif ingress.backend is defined %}
+ backend:
+ serviceName: {{ ingress.backend.service }}
+ servicePort: {{ ingress.backend.port }}
+ {%- endif %}
diff --git a/kubernetes/files/kube-addons/contrail/contrail.yaml b/kubernetes/files/kube-addons/contrail/contrail.yaml
index 2adef76..ee56429 100644
--- a/kubernetes/files/kube-addons/contrail/contrail.yaml
+++ b/kubernetes/files/kube-addons/contrail/contrail.yaml
@@ -1,5 +1,14 @@
{%- from "kubernetes/map.jinja" import common with context -%}
{%- from "kubernetes/map.jinja" import master with context -%}
+
+{%- if master.network.get('opencontrail',{}).get('version', 4.0) >= 4.1 %}
+ {%- set kafka_path = "/etc/kafka" %}
+ {%- set kafka_path_log = "/var/log/kafka" %}
+
+{%- else %}
+ {%- set kafka_path = "/usr/share/kafka/config" %}
+ {%- set kafka_path_log = "/usr/share/kafka/logs" %}
+{%- endif %}
---
apiVersion: apps/v1beta2
@@ -104,6 +113,8 @@
mountPath: /var/log/contrail
- name: journal-controller
mountPath: /var/log/journal
+ - name: var-log-configdb-zookeeper
+ mountPath: /var/log/zookeeper
- name: etc-hostname
mountPath: /etc/hostname
readOnly: true
@@ -132,14 +143,14 @@
mountPath: /var/lib/zookeeper
- name: var-lib-zookeeper-myid
mountPath: /var/lib/zookeeper/myid
- - name: usr-share-kafka-config-server-properties
- mountPath: /usr/share/kafka/config/server.properties
- - name: usr-share-kafka-config-consumer-properties
- mountPath: /usr/share/kafka/config/consumer.properties
- - name: usr-share-kafka-config-zookeeper-properties
- mountPath: /usr/share/kafka/config/zookeeper.properties
- - name: usr-share-kafka-logs
- mountPath: /usr/share/kafka/logs
+ - name: kafka-config-server-properties
+ mountPath: {{ kafka_path }}/server.properties
+ - name: kafka-config-consumer-properties
+ mountPath: {{ kafka_path }}/consumer.properties
+ - name: kafka-config-zookeeper-properties
+ mountPath: {{ kafka_path }}/zookeeper.properties
+ - name: kafka-logs
+ mountPath: {{ kafka_path_log }}
- name: etc-zookeeper-conf-zoo-analytics-cfg
mountPath: /etc/zookeeper/conf/zoo.cfg
- name: etc-zookeeper-conf-log4j-properties
@@ -148,6 +159,8 @@
mountPath: /var/log/contrail
- name: journal-analyticsdb
mountPath: /var/log/journal
+ - name: var-log-analyticsdb-zookeeper
+ mountPath: /var/log/zookeeper
- name: etc-hostname
mountPath: /etc/hostname
readOnly: true
@@ -215,6 +228,10 @@
hostPath:
path: /var/log/journal/contrail-controller
type: DirectoryOrCreate
+ - name: var-log-configdb-zookeeper
+ hostPath:
+ path: /var/log/configdb/zookeeper
+ type: DirectoryOrCreate
# analyticsdb
- name: etc-cassandra-cassandra-env-analytics-sh
@@ -241,21 +258,21 @@
hostPath:
path: /var/lib/zookeeper/myid
type: File
- - name: usr-share-kafka-config-server-properties
+ - name: kafka-config-server-properties
hostPath:
- path: /usr/share/kafka/config/server.properties
+ path: {{ kafka_path }}/server.properties
type: File
- - name: usr-share-kafka-config-consumer-properties
+ - name: kafka-config-consumer-properties
hostPath:
- path: /usr/share/kafka/config/consumer.properties
+ path: {{ kafka_path }}/consumer.properties
type: File
- - name: usr-share-kafka-config-zookeeper-properties
+ - name: kafka-config-zookeeper-properties
hostPath:
- path: /usr/share/kafka/config/zookeeper.properties
+ path: {{ kafka_path }}/zookeeper.properties
type: File
- - name: usr-share-kafka-logs
+ - name: kafka-logs
hostPath:
- path: /usr/share/kafka/logs
+ path: {{ kafka_path_log }}
type: DirectoryOrCreate
- name: etc-zookeeper-conf-zoo-analytics-cfg
hostPath:
@@ -269,6 +286,10 @@
hostPath:
path: /var/log/journal/contrail-analyticsdb
type: DirectoryOrCreate
+ - name: var-log-analyticsdb-zookeeper
+ hostPath:
+ path: /var/log/analyticsdb/zookeeper
+ type: DirectoryOrCreate
# analytics
- name: etc-redis-redis-conf
diff --git a/kubernetes/files/kube-addons/ingress-nginx/ingress-nginx.yaml b/kubernetes/files/kube-addons/ingress-nginx/ingress-nginx.yaml
index aedb8be..7e28da8 100644
--- a/kubernetes/files/kube-addons/ingress-nginx/ingress-nginx.yaml
+++ b/kubernetes/files/kube-addons/ingress-nginx/ingress-nginx.yaml
@@ -86,6 +86,12 @@
kind: ConfigMap
apiVersion: v1
+{%- if common.addons.get('ingress-nginx', {}).tcp_data is defined %}
+data:
+ {%- for key, value in common.addons.get('ingress-nginx').tcp_data.items() %}
+ {{ key }}: "{{ value }}"
+ {%- endfor %}
+{%- endif %}
metadata:
name: tcp-services
namespace: ingress-nginx
@@ -97,6 +103,12 @@
kind: ConfigMap
apiVersion: v1
+{%- if common.addons.get('ingress-nginx', {}).udp_data is defined %}
+data:
+ {%- for key, value in common.addons.get('ingress-nginx').udp_data.items() %}
+ {{ key }}: "{{ value }}"
+ {%- endfor %}
+{%- endif %}
metadata:
name: udp-services
namespace: ingress-nginx
@@ -320,6 +332,18 @@
containerPort: 80
- name: https
containerPort: 443
+ {%- if common.addons.get('ingress-nginx', {}).tcp_ports is defined %}
+ {%- for port in common.addons.get('ingress-nginx').tcp_ports %}
+ - name: tcp{{ port }}
+ containerPort: {{ port }}
+ {%- endfor %}
+ {%- endif %}
+ {%- if common.addons.get('ingress-nginx', {}).udp_ports is defined %}
+ {%- for port in common.addons.get('ingress-nginx').udp_ports %}
+ - name: udp{{ port }}
+ containerPort: {{ port }}
+ {%- endfor %}
+ {%- endif %}
livenessProbe:
failureThreshold: 3
httpGet:
@@ -352,7 +376,7 @@
app.kubernetes.io/part-of: ingress-nginx
addonmanager.kubernetes.io/mode: Reconcile
spec:
- externalTrafficPolicy: Local
+ externalTrafficPolicy: {{ common.addons.get('ingress-nginx', {}).get('externalTrafficPolicy', 'Local') }}
type: LoadBalancer
selector:
app.kubernetes.io/name: ingress-nginx
@@ -364,5 +388,18 @@
- name: https
port: 443
targetPort: https
-
+ {%- if common.addons.get('ingress-nginx', {}).tcp_ports is defined %}
+ {%- for port in common.addons.get('ingress-nginx').tcp_ports %}
+ - name: tcp{{ port }}
+ port: {{ port }}
+ targetPort: tcp{{ port }}
+ {%- endfor %}
+ {%- endif %}
+ {%- if common.addons.get('ingress-nginx', {}).udp_ports is defined %}
+ {%- for port in common.addons.get('ingress-nginx').udp_ports %}
+ - name: udp{{ port }}
+ port: {{ port }}
+ targetPort: udp{{ port }}
+ {%- endfor %}`
+ {%- endif %}
---
diff --git a/kubernetes/files/kubelet/default.pool b/kubernetes/files/kubelet/default.pool
index 8e2a1e9..4ba5328 100644
--- a/kubernetes/files/kubelet/default.pool
+++ b/kubernetes/files/kubelet/default.pool
@@ -56,7 +56,7 @@
{%- endif %}
--file-check-frequency={{ pool.kubelet.frequency }} \
{%- if common.get('cloudprovider', {}).get('enabled') and common.get('cloudprovider', {}).get('provider') == 'openstack' %}
---cloud-provider=openstack \
+--cloud-provider=external \
--cloud-config=/etc/kubernetes/cloud-config \
{%- endif %}
{%- if common.addons.get('virtlet', {}).get('enabled') %}
diff --git a/kubernetes/files/opencontrail/4.1 b/kubernetes/files/opencontrail/4.1
new file mode 120000
index 0000000..0c16b90
--- /dev/null
+++ b/kubernetes/files/opencontrail/4.1
@@ -0,0 +1 @@
+4.0/
\ No newline at end of file
diff --git a/kubernetes/files/rc.yml b/kubernetes/files/rc.yml
index 0a31b6c..8edb271 100644
--- a/kubernetes/files/rc.yml
+++ b/kubernetes/files/rc.yml
@@ -2,10 +2,18 @@
apiVersion: {{ service.apiVersion }}
kind: {{ service.kind }}
metadata:
- name: {{ service.service }}-{{ service.role }}
- namespace: {{ service.namespace }}
+ {%- if service.role is defined %}
labels:
+ name: {{ service.service }}-{{ service.role }}
app: {{ service.service }}-{{ service.role }}
+ name: {{ service.service }}-{{ service.role }}
+ {%- else %}
+ labels:
+ name: {{ service.service }}
+ app: {{ service.service }}
+ name: {{ service.service }}
+ {%- endif %}
+ namespace: {{ service.namespace }}
spec:
replicas: {{ service.replicas }}
{%- if service.kind == 'PetSet' %}
@@ -14,7 +22,11 @@
template:
metadata:
labels:
+ {%- if service.role is defined %}
app: {{ service.service }}-{{ service.role }}
+ {%- else %}
+ app: {{ service.service }}
+ {%- endif %}
annotations:
{%- if service.hostname is defined %}
pod.beta.kubernetes.io/hostname: {{ service.hostname }}
@@ -95,6 +107,16 @@
{%- if service.host_pid is defined %}
hostPID: True
{%- endif %}
+ {%- if service.host_aliases is defined %}
+ hostAliases:
+ {%- for host_alias in service.host_aliases %}
+ - ip: {{ host_alias.ip }}
+ hostnames:
+ {%- for hostname in host_alias.hostnames %}
+ - {{ hostname }}
+ {%- endfor %}
+ {%- endfor %}
+ {%- endif %}
containers:
{%- for container_name, container in service.container.items() %}
- name: {{ container_name }}
diff --git a/kubernetes/files/svc.yml b/kubernetes/files/svc.yml
index b236417..d730639 100644
--- a/kubernetes/files/svc.yml
+++ b/kubernetes/files/svc.yml
@@ -2,10 +2,17 @@
apiVersion: v1
kind: Service
metadata:
+ {%- if service.role is defined %}
labels:
name: {{ service.service }}-{{ service.role }}
app: {{ service.service }}-{{ service.role }}
name: {{ service.service }}-{{ service.role }}
+ {%- else %}
+ labels:
+ name: {{ service.service }}
+ app: {{ service.service }}
+ name: {{ service.service }}
+ {%- endif %}
namespace: {{ service.namespace }}
spec:
ports:
@@ -15,7 +22,11 @@
{%- endfor %}
type: {{ service.type }}
selector:
+ {%- if service.role is defined %}
app: {{ service.service }}-{{ service.role }}
+ {%- else %}
+ app: {{ service.service }}
+ {%- endif %}
{%- if service.cluster_ip is defined %}
clusterIP: {{ service.cluster_ip }}
{%- endif %}
diff --git a/kubernetes/files/systemd/criproxy.service b/kubernetes/files/systemd/criproxy.service
index 9980501..2279313 100644
--- a/kubernetes/files/systemd/criproxy.service
+++ b/kubernetes/files/systemd/criproxy.service
@@ -16,7 +16,7 @@
[Service]
SyslogIdentifier=criproxy
User=root
-ExecStart=/usr/bin/criproxy -alsologtostderr \
+ExecStart=/usr/bin/criproxy -logtostderr \
{%- if common.get('containerd', {}).get('enabled') %}
-connect /run/containerd/containerd.sock,virtlet.cloud:/run/virtlet.sock \
{%- else %}
@@ -24,7 +24,6 @@
{%- endif %}
-listen /var/run/criproxy.sock \
-v 3 \
- -alsologtostderr \
{%- if salt['pkg.version_cmp'](version,'1.8') < 0 %}
-apiVersion {{ version }} \
{%- endif %}
diff --git a/kubernetes/init.sls b/kubernetes/init.sls
index 22aad26..a882a97 100644
--- a/kubernetes/init.sls
+++ b/kubernetes/init.sls
@@ -10,4 +10,7 @@
{%- if pillar.kubernetes.control is defined %}
- kubernetes.control
{%- endif %}
+{%- if pillar.kubernetes.client is defined %}
+- kubernetes.client
+{%- endif %}
{%- endif %}
diff --git a/kubernetes/map.jinja b/kubernetes/map.jinja
index 615404c..3c3d4cb 100644
--- a/kubernetes/map.jinja
+++ b/kubernetes/map.jinja
@@ -111,6 +111,9 @@
},
}, merge=salt['pillar.get']('kubernetes:control')) %}
+{% set client = salt['grains.filter_by']({
+}, merge=salt['pillar.get']('kubernetes:client')) %}
+
{%- set monitoring = salt['grains.filter_by']({
'default': {
'instance_minor_threshold_percent': 0.3,
diff --git a/kubernetes/master/controller.sls b/kubernetes/master/controller.sls
index a458868..0bfe698 100644
--- a/kubernetes/master/controller.sls
+++ b/kubernetes/master/controller.sls
@@ -231,6 +231,11 @@
--root-ca-file=/etc/kubernetes/ssl/ca-{{ master.ca }}.crt
--service-account-private-key-file=/etc/kubernetes/ssl/kubernetes-server.key
--use-service-account-credentials
+{%- if common.get('cloudprovider', {}).get('enabled') and common.get('cloudprovider', {}).get('provider') == 'openstack' %}
+ --external-cloud-volume-plugin=openstack
+ --cloud-config /etc/kubernetes/cloud-config.intree
+ --cloud-provider external
+{%- endif %}
--v={{ master.get('verbosity', 2) }}
{%- if master.network.get('flannel', {}).get('enabled', False) %}
--allocate-node-cidrs=true
@@ -317,11 +322,19 @@
- file: /etc/kubernetes/cloud-config
- file: /etc/default/openstack-cloud-controller-manager
- file: /etc/kubernetes/controller-manager.kubeconfig
+ - file: /usr/bin/openstack-cloud-controller-manager
+
+kube_controller_mnanager_service:
+ service.running:
+ - name: kube-controller-manager
+ - watch:
+ - file: /etc/kubernetes/cloud-config.intree
{%- endif %}
{%- endif %}
{%- endif %}
+{%- if master.namespace is defined %}
{%- for name,namespace in master.namespace.items() %}
@@ -332,7 +345,11 @@
kubernetes_namespace_create_{{ name }}:
cmd.run:
- name: kubectl create ns "{{ name }}"
- - name: kubectl get ns -o=custom-columns=NAME:.metadata.name | grep -v NAME | grep "{{ name }}" > /dev/null || kubectl create ns "{{ name }}"
+ - unless: kubectl get ns -o=custom-columns=NAME:.metadata.name | grep -v NAME | grep "{{ name }}"
+ - retry:
+ attempts: 3
+ until: True
+ interval: 10
{%- if grains.get('noservices') %}
- onlyif: /bin/false
{%- endif %}
@@ -341,12 +358,19 @@
kubernetes_namespace_delete_{{ name }}:
cmd.run:
- - name: kubectl get ns -o=custom-columns=NAME:.metadata.name | grep -v NAME | grep "{{ name }}" > /dev/null && kubectl delete ns "{{ name }} || true"
+ - name: kubectl delete ns "{{ name }}"
+ - onlyif:
+ - kubectl get ns -o=custom-columns=NAME:.metadata.name | grep -v NAME | grep "{{ name }}" > /dev/null
+ {%- if grains.get('noservices') %}
+ - /bin/false
+ {%- endif %}
{%- endif %}
{%- endfor %}
+{%- endif %}
+
{%- if master.registry.secret is defined %}
{%- for name,registry in master.registry.secret.items() %}
diff --git a/kubernetes/meta/fluentd.yml b/kubernetes/meta/fluentd.yml
index 623fcea..3a88acf 100644
--- a/kubernetes/meta/fluentd.yml
+++ b/kubernetes/meta/fluentd.yml
@@ -1,22 +1,21 @@
-{%- from "kubernetes/map.jinja" import common with context -%}
+{%- from "kubernetes/map.jinja" import common, master, pool with context -%}
{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
-{%- from "kubernetes/map.jinja" import pool, master %}
-{%- if pool.get('enabled', False) %}
-{% set network = pool.get('network', {}) %}
-{%- else %}
-{%- if master.get('enabled', False) %}
-{% set network = master.get('network', {}) %}
-{% endif %}
-{% endif %}
+ {%- if pool.get('enabled', False) %}
+ {% set network = pool.get('network', {}) %}
+ {%- else %}
+ {%- if master.get('enabled', False) %}
+ {% set network = master.get('network', {}) %}
+ {% endif %}
+ {% endif %}
-{%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
+ {%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
agent:
plugin:
fluent-plugin-kubernetes_metadata_filter:
deb: ['td-agent-additional-plugins']
config:
label:
- {%- if pillar.docker is defined %}
+ {%- if pillar.docker is defined %}
docker:
filter:
add_drop_tag:
@@ -26,7 +25,7 @@
record:
- name: drop_event
value: ${ record.fetch('attrs', {}).fetch('io.kubernetes.pod.name', '') }
- {%- endif %}
+ {%- endif %}
kubernetes:
input:
container:
@@ -36,10 +35,16 @@
path_key: log_path
pos_file: {{ positiondb }}/kubernetes.pos
parser:
+ {%- if pillar.docker is defined %}
type: json
+ {%- else %}
+ # Containerd CRI log format https://regex101.com/r/BAw3bQ/1
+ type: regexp
+ format: /^(?<time>.+) (?<stream>stdout|stderr)( (?<logtag>.))? (?<Payload>.*)$/
+ {%- endif %}
time_format: '%Y-%m-%dT%H:%M:%S.%NZ'
keep_time_key: false
- {%- if network is defined and network.get('calico', {}).get('enabled', False) %}
+ {%- if network is defined and network.get('calico', {}).get('enabled', False) %}
bird:
type: tail
tag: kubernetes.calico.bird
@@ -76,19 +81,19 @@
time_key: Timestamp
keep_time_key: false
format: '/^(?<Timestamp>[^ ]+ [^ ]+) \[(?<orig_severity_label>[^ ]+)\]\[\d+?\] (?<Payload>.*)$/'
- {%- endif %}
+ {%- endif %}
filter:
add_kubernetes_meta:
tag: 'temp.kubernetes.container.**'
type: kubernetes_metadata
kubernetes_url: https://{{ pool.apiserver.host }}:{{ pool.apiserver.secure_port }}
- {%- if common.get('cloudprovider', {}).get('enabled') and common.get('cloudprovider', {}).get('provider') == 'openstack' %}
+ {%- if common.get('cloudprovider', {}).get('enabled') and common.get('cloudprovider', {}).get('provider') == 'openstack' %}
client_cert: /etc/kubernetes/ssl/kubelet-client-fqdn.crt
client_key: /etc/kubernetes/ssl/kubelet-client-fqdn.key
- {%- else %}
+ {%- else %}
client_cert: /etc/kubernetes/ssl/kubelet-client.crt
client_key: /etc/kubernetes/ssl/kubelet-client.key
- {%- endif %}
+ {%- endif %}
ca_file: /etc/kubernetes/ssl/ca-kubernetes.crt
verify_ssl: True
enrich_container:
@@ -104,7 +109,7 @@
value: 6
- name: programname
value: ${ record['kubernetes']['container_name'] }
- {%- if network is defined and network.get('calico', {}).get('enabled', False) %}
+ {%- if network is defined and network.get('calico', {}).get('enabled', False) %}
enrich_bird:
tag: 'kubernetes.calico.bird'
type: record_transformer
@@ -140,7 +145,7 @@
value: ${ {'DEBUG'=>7,'INFO'=>6,'WARNING'=>4,'ERROR'=>3,'FATAL'=>2,'PANIC'=>1}[record['orig_severity_label']].to_i }
- name: programname
value: calico-felix
- {%- endif %}
+ {%- endif %}
match:
cast_service_tag:
tag: 'temp.kubernetes.container.**'
diff --git a/kubernetes/pool/images.sls b/kubernetes/pool/images.sls
new file mode 100644
index 0000000..f58f547
--- /dev/null
+++ b/kubernetes/pool/images.sls
@@ -0,0 +1,14 @@
+{%- from "kubernetes/map.jinja" import pool with context %}
+{%- if pool.get('enabled', False) and pool.get('images', {}) %}
+
+{%- for image in pool.get('images', []) %}
+
+{{ image }}_image:
+ cmd.run:
+ - name: /usr/local/bin/crictl pull {{ image }}
+ - onlyif: "test -e /usr/local/bin/crictl"
+ - unless: "/usr/local/bin/crictl images -o yaml | grep {{ image }}"
+
+{%- endfor %}
+
+{%- endif %}
\ No newline at end of file
diff --git a/kubernetes/pool/init.sls b/kubernetes/pool/init.sls
index d5acd23..8a0919f 100644
--- a/kubernetes/pool/init.sls
+++ b/kubernetes/pool/init.sls
@@ -23,3 +23,6 @@
{%- if common.addons.get('virtlet', {}).get('use_apparmor') and not pillar.get('kubernetes', {}).get('master', False) %}
- kubernetes.pool.virtlet-apparmor
{%- endif %}
+{%- if pool.get('images', {}) %}
+- kubernetes.pool.images
+{%- endif %}
diff --git a/metadata/service/master/cluster.yml b/metadata/service/master/cluster.yml
index afeb317..c611cda 100644
--- a/metadata/service/master/cluster.yml
+++ b/metadata/service/master/cluster.yml
@@ -47,9 +47,6 @@
ca: kubernetes
storage:
engine: none
- namespace:
- kube-system:
- enabled: True
network:
calico:
etcd:
diff --git a/metadata/service/master/single.yml b/metadata/service/master/single.yml
index be5d701..a9314b1 100644
--- a/metadata/service/master/single.yml
+++ b/metadata/service/master/single.yml
@@ -65,9 +65,6 @@
ca: kubernetes
storage:
engine: none
- namespace:
- kube-system:
- enabled: True
network:
calico:
etcd: