Update k8s-ha-* models
Related-PROD: PROD-25976
Change-Id: Id31e6604c71534086e90b63ae1c0cf3a319ec0b8
diff --git a/classes/cluster/k8s-ha-calico-flannel-virtlet/infra/config.yml b/classes/cluster/k8s-ha-calico-flannel-virtlet/infra/config.yml
index a5eb9ab..0ed5041 100644
--- a/classes/cluster/k8s-ha-calico-flannel-virtlet/infra/config.yml
+++ b/classes/cluster/k8s-ha-calico-flannel-virtlet/infra/config.yml
@@ -1,6 +1,4 @@
classes:
-- system.linux.system.repo.mcp.apt_mirantis.ubuntu
-- system.linux.system.repo.mcp.apt_mirantis.saltstack
- system.openssh.client.lab
- system.salt.master.pkg
- system.salt.minion.ca.salt_master
@@ -8,7 +6,7 @@
- system.salt.minion.cert.k8s_server
- system.reclass.storage.salt
- system.reclass.storage.system.kubernetes_control_cluster
-- cluster.k8s-ha-calico-flannel-virtlet
+- cluster.k8s-ha-calico-flannel-virtlet.infra
parameters:
_param:
salt_master_base_environment: prd
diff --git a/classes/cluster/k8s-ha-calico-flannel-virtlet/infra/init.yml b/classes/cluster/k8s-ha-calico-flannel-virtlet/infra/init.yml
index f3e1fdf..621f3a0 100644
--- a/classes/cluster/k8s-ha-calico-flannel-virtlet/infra/init.yml
+++ b/classes/cluster/k8s-ha-calico-flannel-virtlet/infra/init.yml
@@ -1,9 +1,16 @@
classes:
+- system.defaults
- system.linux.system.single
+- system.linux.system.single.debian
+- system.linux.network.dynamic_hosts
+- system.salt.minion.single
- system.linux.system.repo.mcp.extra
- system.linux.system.repo.mcp.apt_mirantis.saltstack
- system.linux.system.repo.mcp.apt_mirantis.ubuntu
+- system.linux.system.repo.mcp.apt_mirantis.salt-formulas
- system.openssh.server.team.all
+- cluster.k8s-ha-calico-flannel-virtlet.kubernetes
+- cluster.overrides
parameters:
_param:
# infra service addresses
@@ -15,6 +22,11 @@
cluster_name: k8s-ha-calico-flannel-virtlet
apt_mk_version: nightly
primary_interface: ens3
+ control_interface: ens4
+
+ dns_server01: 172.18.176.6
+ dns_server02: 172.17.44.91
+
linux:
network:
host:
diff --git a/classes/cluster/k8s-ha-calico-flannel-virtlet/init.yml b/classes/cluster/k8s-ha-calico-flannel-virtlet/init.yml
deleted file mode 100644
index f3f2d2a..0000000
--- a/classes/cluster/k8s-ha-calico-flannel-virtlet/init.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-classes:
-- system.defaults
-- system.linux.network.dynamic_hosts
-- cluster.k8s-ha-calico-flannel-virtlet.kubernetes
-- cluster.k8s-ha-calico-flannel-virtlet.infra
-- cluster.overrides
diff --git a/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/common.yml b/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/common.yml
new file mode 100644
index 0000000..7b286d0
--- /dev/null
+++ b/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/common.yml
@@ -0,0 +1,67 @@
+classes:
+- system.linux.system.repo.mcp.apt_mirantis.docker
+- system.kubernetes.pool.cluster
+- system.salt.minion.cert.k8s_client
+- system.salt.minion.cert.etcd_client
+
+parameters:
+ kubernetes:
+ common:
+ addons:
+ virtlet:
+ enabled: ${_param:kubernetes_virtlet_enabled}
+ namespace: ${_param:kubernetes_addon_namespace}
+ image: ${_param:kubernetes_virtlet_image}
+ criproxy_version: ${_param:kubernetes_criproxy_version}
+ criproxy_source: ${_param:kubernetes_criproxy_checksum}
+ hosts:
+ - ${_param:kubernetes_compute01_hostname}
+ dashboard:
+ enabled: ${_param:kubernetes_dashboard}
+ image: ${_param:kubernetes_dashboard_image}
+ helm:
+ enabled: ${_param:kubernetes_helm_enabled}
+ netchecker:
+ enabled: ${_param:kubernetes_netchecker_enabled}
+ agent_probeurls: ${_param:kubernetes_netchecker_agent_probeurls}
+ externaldns:
+ enabled: ${_param:kubernetes_externaldns_enabled}
+ image: ${_param:kubernetes_externaldns_image}
+ provider: ${_param:kubernetes_externaldns_provider}
+ metallb:
+ enabled: ${_param:kubernetes_metallb_enabled}
+ addresses:
+ - ${_param:kubernetes_metallb_addresses_pool}
+ ingress-nginx:
+ enabled: ${_param:kubernetes_ingressnginx_enabled}
+ metrics-server:
+ enabled: ${_param:kubernetes_metrics_server_enabled}
+ hyperkube:
+ source: ${_param:kubernetes_hyperkube_source}
+ source_hash: ${_param:kubernetes_hyperkube_source_hash}
+ pause_image: ${_param:kubernetes_pause_image}
+ pool:
+ proxy:
+ daemon_opts:
+ cluster-cidr: ${_param:calico_private_network}/${_param:calico_private_netmask}
+ network:
+ calico:
+ no_default_pools: False
+ etcd:
+ ssl:
+ enabled: True
+ policy:
+ enabled: ${_param:kubernetes_calico_policy_enabled}
+ linux:
+ system:
+ kernel:
+ sysctl:
+ # The default operating system limits on mmap counts is likely to be too low, which may result in out of memory exceptions.
+ vm.max_map_count: 262144
+ storage:
+ enabled: true
+ swap:
+ img:
+ enabled: false
+ engine: file
+ device: /swap.img
diff --git a/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/compute.yml b/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/compute.yml
index ead41e5..1b781a2 100644
--- a/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/compute.yml
+++ b/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/compute.yml
@@ -1,26 +1,21 @@
classes:
-- system.linux.system.repo.mcp.apt_mirantis.docker
-- system.kubernetes.pool.cluster
- system.linux.network.hosts
-- system.salt.minion.cert.k8s_client
-- system.salt.minion.cert.etcd_client
-- cluster.k8s-ha-calico-flannel-virtlet
+- cluster.k8s-ha-calico-flannel-virtlet.kubernetes.common
+- cluster.k8s-ha-calico-flannel-virtlet.kubernetes.networking.virtual
+- cluster.k8s-ha-calico-flannel-virtlet.infra
+
parameters:
kubernetes:
pool:
- proxy:
- daemon_opts:
- cluster-cidr: ${_param:calico_private_network}/${_param:calico_private_netmask}
kubelet:
address: ${_param:single_address}
fail_on_swap: ${_param:kubelet_fail_on_swap}
network:
genie:
- enabled: true
+ enabled: ${_param:kubernetes_genie_enabled}
default_plugin: calico
- image: ${_param:kubernetes_genie_image}
flannel:
- enabled: true
+ enabled: ${_param:kubernetes_flannel_enabled}
calico:
enabled: true
image: ${_param:kubernetes_calico_image}
@@ -32,44 +27,3 @@
enabled: true
policy:
enabled: ${_param:kubernetes_calico_policy_enabled}
- common:
- addons:
- virtlet:
- enabled: ${_param:kubernetes_virtlet_enabled}
- namespace: ${_param:kubernetes_addon_namespace}
- image: ${_param:kubernetes_virtlet_image}
- criproxy_version: ${_param:kubernetes_criproxy_version}
- criproxy_source: ${_param:kubernetes_criproxy_checksum}
- hosts:
- - ${_param:kubernetes_compute01_hostname}
- hyperkube:
- image: ${_param:kubernetes_hyperkube_image}
- pause_image: ${_param:kubernetes_pause_image}
- linux:
- network:
- resolv:
- dns:
- - 172.18.208.44
- - 172.18.176.4
- - 10.254.0.10
- search:
- - ${_param:cluster_domain}
- interface:
- ens3:
- enabled: true
- type: eth
- proto: static
- name: ${_param:primary_interface}
- address: ${_param:deploy_address}
- netmask: 255.255.255.0
- gateway: 192.168.10.1
- post_up_cmds:
- - ip r rep 10.254.0.0/16 via ${_param:deploy_address}
- ens4:
- enabled: true
- type: eth
- proto: static
- address: ${_param:single_address}
- netmask: '255.255.255.0'
- ipflush_onchange: true
- restart_on_ipflush: true
diff --git a/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/control.yml b/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/control.yml
index c8c9177..d0c621b 100644
--- a/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/control.yml
+++ b/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/control.yml
@@ -1,16 +1,15 @@
classes:
- system.etcd.server.cluster
-- service.kubernetes.control.cluster
- system.haproxy.proxy.listen.kubernetes.apiserver
- system.keepalived.cluster.instance.kube_api_server_vip
-- system.linux.system.repo.docker
- system.salt.minion.cert.etcd_server
- system.kubernetes.master.cluster
- system.kubernetes.master.auth.rbac
- system.kubernetes.control.roles.cluster-admin
- system.kubernetes.control.roles.genie-pod-patch
-- cluster.k8s-ha-calico-flannel-virtlet.kubernetes.compute
-- cluster.k8s-ha-calico-flannel-virtlet
+- cluster.k8s-ha-calico-flannel-virtlet.kubernetes.common
+- cluster.k8s-ha-calico-flannel-virtlet.kubernetes.networking.virtual
+- cluster.k8s-ha-calico-flannel-virtlet.infra
parameters:
_param:
keepalived_k8s_apiserver_vip_interface: ${_param:primary_interface}
@@ -23,38 +22,9 @@
key: /calico/ipam/v2/assignment/ipv4/block/${_param:calico_private_network}-${_param:calico_private_netmask}
value: '{"masquerade":true,"cidr":"${_param:calico_private_network}/${_param:calico_private_netmask}"}'
kubernetes:
- common:
- addons:
- dashboard:
- enabled: ${_param:kubernetes_dashboard}
- image: ${_param:kubernetes_dashboard_image}
- helm:
- enabled: ${_param:kubernetes_helm_enabled}
- netchecker:
- enabled: ${_param:kubernetes_netchecker_enabled}
- agent_probeurls: ${_param:kubernetes_netchecker_agent_probeurls}
- dns:
- enabled: ${_param:kubernetes_dns}
- kubedns_image: ${_param:kubernetes_kubedns_image}
- dnsmasq_image: ${_param:kubernetes_dnsmasq_image}
- sidecar_image: ${_param:kubernetes_sidecar_image}
- autoscaler:
- image: ${_param:kubernetes_dns_autoscaler_image}
- coredns:
- enabled: ${_param:kubernetes_coredns_enabled}
- externaldns:
- enabled: ${_param:kubernetes_externaldns_enabled}
- image: ${_param:kubernetes_externaldns_image}
- provider: ${_param:kubernetes_externaldns_provider}
- metallb:
- enabled: ${_param:kubernetes_metallb_enabled}
- addresses:
- - ${_param:kubernetes_metallb_addresses_pool}
- ingress-nginx:
- enabled: ${_param:kubernetes_ingressnginx_enabled}
- metrics-server:
- enabled: ${_param:kubernetes_metrics_server_enabled}
master:
+ apiserver:
+ insecure_address: 0.0.0.0
kubelet:
address: ${_param:single_address}
fail_on_swap: ${_param:kubelet_fail_on_swap}
@@ -63,11 +33,10 @@
enabled: true
network:
genie:
- enabled: true
+ enabled: ${_param:kubernetes_genie_enabled}
default_plugin: calico
- image: ${_param:kubernetes_genie_image}
flannel:
- enabled: true
+ enabled: ${_param:kubernetes_flannel_enabled}
private_ip_range: 10.20.0.0/16
calico:
enabled: true
diff --git a/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/init.yml b/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/init.yml
index 41c03a8..615d818 100644
--- a/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/init.yml
+++ b/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/init.yml
@@ -1,5 +1,7 @@
parameters:
_param:
+ # General
+ linux_system_codename: xenial
salt_minion_ca_host: ${_param:infra_config_hostname}.${_param:cluster_domain}
salt_minion_ca_authority: salt_master_ca
@@ -18,29 +20,21 @@
etcd_initial_token: IN7KaRMSo3xkGxkjAAPtkRkAgqN4ZNRq
# component docker images
- kubernetes_pause_image: docker-prod-virtual.docker.mirantis.net/mirantis/kubernetes/pause-amd64:v1.11.3-2
+ kubernetes_hyperkube_source: https://docker-prod-local.artifactory.mirantis.com/artifactory/binary-prod-local/mirantis/kubernetes/hyperkube-binaries/hyperkube_v1.12.3-2_1544133573591
+ kubernetes_hyperkube_source_hash: md5=fc23eaf3ba63d9ed9d141f465f584012
+ kubernetes_pause_image: docker-prod-virtual.docker.mirantis.net/mirantis/kubernetes/pause-amd64:v1.12.3-2
kubernetes_virtlet_image: mirantis/virtlet:v1.4.1
kubernetes_criproxy_version: v0.12.0
kubernetes_criproxy_checksum: md5=371cacd3d8568eb88425498b48a649dd
kubernetes_netchecker_agent_probeurls: "http://ipinfo.io"
- kubernetes_genie_image: mirantis/cnigenie:latest
# kube-controllers image is temporarily added here as it's not defined in TCP-QA by now
kubernetes_calico_kube_controllers_image: docker-prod-virtual.docker.mirantis.net/mirantis/projectcalico/calico/kube-controllers:v3.1.3
- # switches of addons
- kubernetes_addon_namespace: kube-system
- kubernetes_dns: false
- kubernetes_dashboard: true
- kubernetes_helm_enabled: false
- kubernetes_netchecker_enabled: true
- kubernetes_calico_policy_enabled: false
- kubernetes_virtlet_enabled: true
- kubernetes_containerd_enabled: true
-
# addresses and hostnames
kubernetes_internal_api_address: 10.254.0.1
+ kubernetes_internal_dns_address: 10.254.0.10
kubernetes_control_hostname: ctl
- kubernetes_control_address: 192.168.10.253
+ kubernetes_control_address: 172.16.10.253
kubernetes_control_node01_hostname: ctl01
kubernetes_control_node02_hostname: ctl02
kubernetes_control_node03_hostname: ctl03
@@ -48,22 +42,19 @@
kubernetes_control_node01_address: 172.16.10.101
kubernetes_control_node02_address: 172.16.10.102
kubernetes_control_node03_address: 172.16.10.103
- kubernetes_control_node01_deploy_address: 192.168.10.101
- kubernetes_control_node02_deploy_address: 192.168.10.102
- kubernetes_control_node03_deploy_address: 192.168.10.103
cluster_vip_address: ${_param:kubernetes_control_address}
- cluster_local_address: ${_param:deploy_address}
+ cluster_local_address: ${_param:single_address}
# etcd stuff
cluster_node01_hostname: ${_param:kubernetes_control_node01_hostname}
- cluster_node01_address: ${_param:kubernetes_control_node01_deploy_address}
+ cluster_node01_address: ${_param:kubernetes_control_node01_address}
cluster_node01_port: 4001
cluster_node02_hostname: ${_param:kubernetes_control_node02_hostname}
- cluster_node02_address: ${_param:kubernetes_control_node02_deploy_address}
+ cluster_node02_address: ${_param:kubernetes_control_node02_address}
cluster_node02_port: 4001
cluster_node03_hostname: ${_param:kubernetes_control_node03_hostname}
- cluster_node03_address: ${_param:kubernetes_control_node03_deploy_address}
+ cluster_node03_address: ${_param:kubernetes_control_node03_address}
cluster_node03_port: 4001
# calico
@@ -77,14 +68,44 @@
kubernetes_metallb_addresses_pool: 172.16.10.60-172.16.10.80
# switches of addons
- kubernetes_coredns_enabled: true
- kubernetes_externaldns_enabled: false
- kubernetes_metallb_enabled: false
- kubernetes_ingressnginx_enabled: false
- kubernetes_metrics_server_enabled: false
+ kubernetes_kubedns_enabled: False
+ kubernetes_externaldns_enabled: False
+ kubernetes_coredns_enabled: True
+ kubernetes_dashboard: True
+ kubernetes_virtlet_enabled: True
+ kubernetes_flannel_enabled: True
+ kubernetes_genie_enabled: True
+ kubernetes_calico_enabled: True
+ kubernetes_opencontrail_enabled: False
+ kubernetes_contrail_network_controller_enabled: False
+ kubernetes_metallb_enabled: False
+ kubernetes_ingressnginx_enabled: False
+ kubernetes_rbd_enabled: False
+ kubernetes_helm_enabled: False
+ kubernetes_netchecker_enabled: True
+ kubernetes_calico_policy_enabled: False
+ kubernetes_metrics_server_enabled: False
+
+ kubernetes_ingressnginx_controller_replicas: 1
+ kubernetes_virtlet_use_apparmor: False
+
+ kubernetes_addon_namespace: kube-system
+
+
+ # Cloud providers parameters
+
+ kubernetes_cloudprovider_enabled: False
+ kubernetes_cloudprovider_type: 'openstack'
linux:
network:
+ resolv:
+ dns:
+ - ${_param:dns_server01}
+ - ${_param:dns_server02}
+ - ${_param:kubernetes_internal_dns_address}
+ search:
+ - ${_param:cluster_domain}
host:
ctl:
address: ${_param:kubernetes_control_address}
diff --git a/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/networking/virtual.yml b/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/networking/virtual.yml
new file mode 100644
index 0000000..5f93d04
--- /dev/null
+++ b/classes/cluster/k8s-ha-calico-flannel-virtlet/kubernetes/networking/virtual.yml
@@ -0,0 +1,22 @@
+# Virtual node default networking template
+#
+parameters:
+ linux:
+ network:
+ interface:
+ deploy_interface:
+ name: ${_param:primary_interface}
+ enabled: true
+ type: eth
+ proto: dhcp
+ control_interface:
+ name: ${_param:control_interface}
+ enabled: true
+ type: eth
+ proto: static
+ address: ${_param:single_address}
+ netmask: 255.255.255.0
+ ipflush_onchange: true
+ restart_on_ipflush: true
+ post_up_cmds:
+ - ip r rep 10.254.0.0/16 via ${_param:single_address}
diff --git a/classes/cluster/k8s-ha-calico/infra/config.yml b/classes/cluster/k8s-ha-calico/infra/config.yml
index c505d53..d582161 100644
--- a/classes/cluster/k8s-ha-calico/infra/config.yml
+++ b/classes/cluster/k8s-ha-calico/infra/config.yml
@@ -1,6 +1,4 @@
classes:
-- system.linux.system.repo.mcp.apt_mirantis.ubuntu
-- system.linux.system.repo.mcp.apt_mirantis.saltstack
- system.openssh.client.lab
- system.salt.master.pkg
- system.salt.minion.ca.salt_master
@@ -8,7 +6,7 @@
- system.salt.minion.cert.k8s_server
- system.reclass.storage.salt
- system.reclass.storage.system.kubernetes_control_cluster
-- cluster.k8s-ha-calico
+- cluster.k8s-ha-calico.infra
parameters:
_param:
salt_master_base_environment: prd
diff --git a/classes/cluster/k8s-ha-calico/infra/init.yml b/classes/cluster/k8s-ha-calico/infra/init.yml
index fc7bd2e..7f35825 100644
--- a/classes/cluster/k8s-ha-calico/infra/init.yml
+++ b/classes/cluster/k8s-ha-calico/infra/init.yml
@@ -1,9 +1,16 @@
classes:
+- system.defaults
- system.linux.system.single
+- system.linux.system.single.debian
+- system.linux.network.dynamic_hosts
+- system.salt.minion.single
- system.linux.system.repo.mcp.extra
- system.linux.system.repo.mcp.apt_mirantis.saltstack
- system.linux.system.repo.mcp.apt_mirantis.ubuntu
+- system.linux.system.repo.mcp.apt_mirantis.salt-formulas
- system.openssh.server.team.all
+- cluster.k8s-ha-calico.kubernetes
+- cluster.overrides
parameters:
_param:
# infra service addresses
@@ -15,6 +22,11 @@
cluster_name: k8s-ha-calico
apt_mk_version: nightly
primary_interface: ens3
+ control_interface: ens4
+
+ dns_server01: 172.18.176.6
+ dns_server02: 172.17.44.91
+
linux:
network:
host:
diff --git a/classes/cluster/k8s-ha-calico/init.yml b/classes/cluster/k8s-ha-calico/init.yml
deleted file mode 100644
index 148f88e..0000000
--- a/classes/cluster/k8s-ha-calico/init.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-classes:
-- system.defaults
-- system.linux.network.dynamic_hosts
-- cluster.k8s-ha-calico.kubernetes
-- cluster.k8s-ha-calico.infra
-- cluster.overrides
diff --git a/classes/cluster/k8s-ha-calico/kubernetes/common.yml b/classes/cluster/k8s-ha-calico/kubernetes/common.yml
new file mode 100644
index 0000000..7b286d0
--- /dev/null
+++ b/classes/cluster/k8s-ha-calico/kubernetes/common.yml
@@ -0,0 +1,67 @@
+classes:
+- system.linux.system.repo.mcp.apt_mirantis.docker
+- system.kubernetes.pool.cluster
+- system.salt.minion.cert.k8s_client
+- system.salt.minion.cert.etcd_client
+
+parameters:
+ kubernetes:
+ common:
+ addons:
+ virtlet:
+ enabled: ${_param:kubernetes_virtlet_enabled}
+ namespace: ${_param:kubernetes_addon_namespace}
+ image: ${_param:kubernetes_virtlet_image}
+ criproxy_version: ${_param:kubernetes_criproxy_version}
+ criproxy_source: ${_param:kubernetes_criproxy_checksum}
+ hosts:
+ - ${_param:kubernetes_compute01_hostname}
+ dashboard:
+ enabled: ${_param:kubernetes_dashboard}
+ image: ${_param:kubernetes_dashboard_image}
+ helm:
+ enabled: ${_param:kubernetes_helm_enabled}
+ netchecker:
+ enabled: ${_param:kubernetes_netchecker_enabled}
+ agent_probeurls: ${_param:kubernetes_netchecker_agent_probeurls}
+ externaldns:
+ enabled: ${_param:kubernetes_externaldns_enabled}
+ image: ${_param:kubernetes_externaldns_image}
+ provider: ${_param:kubernetes_externaldns_provider}
+ metallb:
+ enabled: ${_param:kubernetes_metallb_enabled}
+ addresses:
+ - ${_param:kubernetes_metallb_addresses_pool}
+ ingress-nginx:
+ enabled: ${_param:kubernetes_ingressnginx_enabled}
+ metrics-server:
+ enabled: ${_param:kubernetes_metrics_server_enabled}
+ hyperkube:
+ source: ${_param:kubernetes_hyperkube_source}
+ source_hash: ${_param:kubernetes_hyperkube_source_hash}
+ pause_image: ${_param:kubernetes_pause_image}
+ pool:
+ proxy:
+ daemon_opts:
+ cluster-cidr: ${_param:calico_private_network}/${_param:calico_private_netmask}
+ network:
+ calico:
+ no_default_pools: False
+ etcd:
+ ssl:
+ enabled: True
+ policy:
+ enabled: ${_param:kubernetes_calico_policy_enabled}
+ linux:
+ system:
+ kernel:
+ sysctl:
+ # The default operating system limits on mmap counts is likely to be too low, which may result in out of memory exceptions.
+ vm.max_map_count: 262144
+ storage:
+ enabled: true
+ swap:
+ img:
+ enabled: false
+ engine: file
+ device: /swap.img
diff --git a/classes/cluster/k8s-ha-calico/kubernetes/compute.yml b/classes/cluster/k8s-ha-calico/kubernetes/compute.yml
index 994ae35..a82e22c 100644
--- a/classes/cluster/k8s-ha-calico/kubernetes/compute.yml
+++ b/classes/cluster/k8s-ha-calico/kubernetes/compute.yml
@@ -1,16 +1,12 @@
classes:
-- system.linux.system.repo.docker
-- system.kubernetes.pool.cluster
- system.linux.network.hosts
-- system.salt.minion.cert.k8s_client
-- system.salt.minion.cert.etcd_client
-- cluster.k8s-ha-calico
+- cluster.k8s-ha-calico.kubernetes.common
+- cluster.k8s-ha-calico.kubernetes.networking.virtual
+- cluster.k8s-ha-calico.infra
+
parameters:
kubernetes:
pool:
- proxy:
- daemon_opts:
- cluster-cidr: ${_param:calico_private_network}/${_param:calico_private_netmask}
kubelet:
address: ${_param:single_address}
fail_on_swap: ${_param:kubelet_fail_on_swap}
@@ -26,44 +22,3 @@
enabled: true
policy:
enabled: ${_param:kubernetes_calico_policy_enabled}
- common:
- addons:
- virtlet:
- enabled: ${_param:kubernetes_virtlet_enabled}
- namespace: ${_param:kubernetes_addon_namespace}
- image: ${_param:kubernetes_virtlet_image}
- criproxy_version: ${_param:kubernetes_criproxy_version}
- criproxy_source: ${_param:kubernetes_criproxy_checksum}
- hosts:
- - ${_param:kubernetes_compute01_hostname}
- hyperkube:
- image: ${_param:kubernetes_hyperkube_image}
- pause_image: ${_param:kubernetes_pause_image}
- linux:
- network:
- resolv:
- dns:
- - 172.18.208.44
- - 172.18.176.4
- - 10.254.0.10
- search:
- - ${_param:cluster_domain}
- interface:
- ens3:
- enabled: true
- type: eth
- proto: static
- name: ${_param:primary_interface}
- address: ${_param:deploy_address}
- netmask: 255.255.255.0
- gateway: 192.168.10.1
- post_up_cmds:
- - ip r rep 10.254.0.0/16 via ${_param:deploy_address}
- ens4:
- enabled: true
- type: eth
- proto: static
- address: ${_param:single_address}
- netmask: '255.255.255.0'
- ipflush_onchange: true
- restart_on_ipflush: true
diff --git a/classes/cluster/k8s-ha-calico/kubernetes/control.yml b/classes/cluster/k8s-ha-calico/kubernetes/control.yml
index ee73105..ba5f7c0 100644
--- a/classes/cluster/k8s-ha-calico/kubernetes/control.yml
+++ b/classes/cluster/k8s-ha-calico/kubernetes/control.yml
@@ -1,18 +1,17 @@
classes:
- system.etcd.server.cluster
-- service.kubernetes.control.cluster
- system.haproxy.proxy.listen.kubernetes.apiserver
- system.keepalived.cluster.instance.kube_api_server_vip
-- system.linux.system.repo.docker
- system.salt.minion.cert.etcd_server
- system.kubernetes.master.cluster
- system.kubernetes.master.auth.rbac
- system.kubernetes.control.roles.cluster-admin
-- cluster.k8s-ha-calico.kubernetes.compute
-- cluster.k8s-ha-calico
+- cluster.k8s-ha-calico.kubernetes.common
+- cluster.k8s-ha-calico.kubernetes.networking.virtual
+- cluster.k8s-ha-calico.infra
parameters:
_param:
- keepalived_k8s_apiserver_vip_interface: ${_param:primary_interface}
+ keepalived_k8s_apiserver_vip_interface: ${_param:control_interface}
keepalived_k8s_apiserver_vip_address: ${_param:kubernetes_control_address}
keepalived_k8s_apiserver_vip_password: password
etcd:
@@ -22,38 +21,9 @@
key: /calico/ipam/v2/assignment/ipv4/block/${_param:calico_private_network}-${_param:calico_private_netmask}
value: '{"masquerade":true,"cidr":"${_param:calico_private_network}/${_param:calico_private_netmask}"}'
kubernetes:
- common:
- addons:
- dashboard:
- enabled: ${_param:kubernetes_dashboard}
- image: ${_param:kubernetes_dashboard_image}
- helm:
- enabled: ${_param:kubernetes_helm_enabled}
- netchecker:
- enabled: ${_param:kubernetes_netchecker_enabled}
- agent_probeurls: ${_param:kubernetes_netchecker_agent_probeurls}
- dns:
- enabled: ${_param:kubernetes_dns}
- kubedns_image: ${_param:kubernetes_kubedns_image}
- dnsmasq_image: ${_param:kubernetes_dnsmasq_image}
- sidecar_image: ${_param:kubernetes_sidecar_image}
- autoscaler:
- image: ${_param:kubernetes_dns_autoscaler_image}
- coredns:
- enabled: ${_param:kubernetes_coredns_enabled}
- externaldns:
- enabled: ${_param:kubernetes_externaldns_enabled}
- image: ${_param:kubernetes_externaldns_image}
- provider: ${_param:kubernetes_externaldns_provider}
- metallb:
- enabled: ${_param:kubernetes_metallb_enabled}
- addresses:
- - ${_param:kubernetes_metallb_addresses_pool}
- ingress-nginx:
- enabled: ${_param:kubernetes_ingressnginx_enabled}
- metrics-server:
- enabled: ${_param:kubernetes_metrics_server_enabled}
master:
+ apiserver:
+ insecure_address: 0.0.0.0
kubelet:
address: ${_param:single_address}
fail_on_swap: ${_param:kubelet_fail_on_swap}
diff --git a/classes/cluster/k8s-ha-calico/kubernetes/init.yml b/classes/cluster/k8s-ha-calico/kubernetes/init.yml
index 62ae9a1..97181e9 100644
--- a/classes/cluster/k8s-ha-calico/kubernetes/init.yml
+++ b/classes/cluster/k8s-ha-calico/kubernetes/init.yml
@@ -1,5 +1,7 @@
parameters:
_param:
+ # General
+ linux_system_codename: xenial
salt_minion_ca_host: ${_param:infra_config_hostname}.${_param:cluster_domain}
salt_minion_ca_authority: salt_master_ca
@@ -18,7 +20,9 @@
etcd_initial_token: IN7KaRMSo3xkGxkjAAPtkRkAgqN4ZNRq
# component docker images
- kubernetes_pause_image: docker-prod-virtual.docker.mirantis.net/mirantis/kubernetes/pause-amd64:v1.11.3-2
+ kubernetes_hyperkube_source: https://docker-prod-local.artifactory.mirantis.com/artifactory/binary-prod-local/mirantis/kubernetes/hyperkube-binaries/hyperkube_v1.12.3-2_1544133573591
+ kubernetes_hyperkube_source_hash: md5=fc23eaf3ba63d9ed9d141f465f584012
+ kubernetes_pause_image: docker-prod-virtual.docker.mirantis.net/mirantis/kubernetes/pause-amd64:v1.12.3-2
kubernetes_virtlet_image: mirantis/virtlet:v1.4.1
kubernetes_criproxy_version: v0.12.0
kubernetes_criproxy_checksum: md5=371cacd3d8568eb88425498b48a649dd
@@ -26,19 +30,11 @@
# kube-controllers image is temporarily added here as it's not defined in TCP-QA by now
kubernetes_calico_kube_controllers_image: docker-prod-virtual.docker.mirantis.net/mirantis/projectcalico/calico/kube-controllers:v3.1.3
- # switches of addons
- kubernetes_addon_namespace: kube-system
- kubernetes_dns: false
- kubernetes_dashboard: true
- kubernetes_helm_enabled: false
- kubernetes_netchecker_enabled: true
- kubernetes_calico_policy_enabled: false
- kubernetes_virtlet_enabled: false
-
# addresses and hostnames
kubernetes_internal_api_address: 10.254.0.1
+ kubernetes_internal_dns_address: 10.254.0.10
kubernetes_control_hostname: ctl
- kubernetes_control_address: 192.168.10.253
+ kubernetes_control_address: 172.16.10.253
kubernetes_control_node01_hostname: ctl01
kubernetes_control_node02_hostname: ctl02
kubernetes_control_node03_hostname: ctl03
@@ -46,22 +42,19 @@
kubernetes_control_node01_address: 172.16.10.101
kubernetes_control_node02_address: 172.16.10.102
kubernetes_control_node03_address: 172.16.10.103
- kubernetes_control_node01_deploy_address: 192.168.10.101
- kubernetes_control_node02_deploy_address: 192.168.10.102
- kubernetes_control_node03_deploy_address: 192.168.10.103
cluster_vip_address: ${_param:kubernetes_control_address}
- cluster_local_address: ${_param:deploy_address}
+ cluster_local_address: ${_param:single_address}
# etcd stuff
cluster_node01_hostname: ${_param:kubernetes_control_node01_hostname}
- cluster_node01_address: ${_param:kubernetes_control_node01_deploy_address}
+ cluster_node01_address: ${_param:kubernetes_control_node01_address}
cluster_node01_port: 4001
cluster_node02_hostname: ${_param:kubernetes_control_node02_hostname}
- cluster_node02_address: ${_param:kubernetes_control_node02_deploy_address}
+ cluster_node02_address: ${_param:kubernetes_control_node02_address}
cluster_node02_port: 4001
cluster_node03_hostname: ${_param:kubernetes_control_node03_hostname}
- cluster_node03_address: ${_param:kubernetes_control_node03_deploy_address}
+ cluster_node03_address: ${_param:kubernetes_control_node03_address}
cluster_node03_port: 4001
# calico
@@ -75,14 +68,44 @@
kubernetes_metallb_addresses_pool: 172.16.10.60-172.16.10.80
# switches of addons
- kubernetes_coredns_enabled: true
- kubernetes_externaldns_enabled: false
- kubernetes_metallb_enabled: false
- kubernetes_ingressnginx_enabled: false
- kubernetes_metrics_server_enabled: false
+ kubernetes_kubedns_enabled: False
+ kubernetes_externaldns_enabled: False
+ kubernetes_coredns_enabled: True
+ kubernetes_dashboard: True
+ kubernetes_virtlet_enabled: False
+ kubernetes_flannel_enabled: False
+ kubernetes_genie_enabled: False
+ kubernetes_calico_enabled: True
+ kubernetes_opencontrail_enabled: False
+ kubernetes_contrail_network_controller_enabled: False
+ kubernetes_metallb_enabled: False
+ kubernetes_ingressnginx_enabled: False
+ kubernetes_rbd_enabled: False
+ kubernetes_helm_enabled: False
+ kubernetes_netchecker_enabled: True
+ kubernetes_calico_policy_enabled: False
+ kubernetes_metrics_server_enabled: False
+
+ kubernetes_ingressnginx_controller_replicas: 1
+ kubernetes_virtlet_use_apparmor: False
+
+ kubernetes_addon_namespace: kube-system
+
+
+ # Cloud providers parameters
+
+ kubernetes_cloudprovider_enabled: False
+ kubernetes_cloudprovider_type: 'openstack'
linux:
network:
+ resolv:
+ dns:
+ - ${_param:dns_server01}
+ - ${_param:dns_server02}
+ - ${_param:kubernetes_internal_dns_address}
+ search:
+ - ${_param:cluster_domain}
host:
ctl:
address: ${_param:kubernetes_control_address}
diff --git a/classes/cluster/k8s-ha-calico/kubernetes/networking/virtual.yml b/classes/cluster/k8s-ha-calico/kubernetes/networking/virtual.yml
new file mode 100644
index 0000000..5f93d04
--- /dev/null
+++ b/classes/cluster/k8s-ha-calico/kubernetes/networking/virtual.yml
@@ -0,0 +1,22 @@
+# Virtual node default networking template
+#
+parameters:
+ linux:
+ network:
+ interface:
+ deploy_interface:
+ name: ${_param:primary_interface}
+ enabled: true
+ type: eth
+ proto: dhcp
+ control_interface:
+ name: ${_param:control_interface}
+ enabled: true
+ type: eth
+ proto: static
+ address: ${_param:single_address}
+ netmask: 255.255.255.0
+ ipflush_onchange: true
+ restart_on_ipflush: true
+ post_up_cmds:
+ - ip r rep 10.254.0.0/16 via ${_param:single_address}