add kitchen tests
more of test suites will be added later
Change-Id: I279def2e4b28d64a75a6978ca9c048a05cd588c2
diff --git a/.kitchen.yml b/.kitchen.yml
new file mode 100644
index 0000000..c9cc946
--- /dev/null
+++ b/.kitchen.yml
@@ -0,0 +1,62 @@
+---
+driver:
+ name: docker
+ hostname: kubernetes
+ use_sudo: true
+
+provisioner:
+ name: salt_solo
+ salt_install: bootstrap
+ salt_bootstrap_url: https://bootstrap.saltstack.com
+ salt_version: latest
+ require_chef: false
+ log_level: error
+ formula: kubernetes
+ grains:
+ noservices: True
+ dependencies:
+ - name: linux
+ repo: git
+ source: https://github.com/salt-formulas/salt-formula-linux
+ state_top:
+ base:
+ "*":
+ - linux
+ - kubernetes
+ pillars:
+ top.sls:
+ base:
+ "*":
+ - linux_repo_docker
+ - linux
+ - kubernetes
+ linux.sls:
+ linux:
+ system:
+ enabled: true
+ name: kubernetes
+ pillars-from-files:
+ linux_repo_docker.sls: tests/pillar/repo_docker.sls
+
+verifier:
+ name: inspec
+ sudo: true
+
+platforms:
+ - name: <%= ENV['PLATFORM'] || 'ubuntu-xenial' %>
+ driver_config:
+ image: <%= ENV['PLATFORM'] || 'trevorj/salty-whales:xenial' %>
+ platform: ubuntu
+
+suites:
+
+ - name: master_cluster
+ provisioner:
+ pillars-from-files:
+ kubernetes.sls: tests/pillar/master_cluster.sls
+ - name: pool_cluster
+ provisioner:
+ pillars-from-files:
+ kubernetes.sls: tests/pillar/pool_cluster.sls
+
+# vim: ft=yaml sw=2 ts=2 sts=2 tw=125
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..a34a650
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,37 @@
+sudo: required
+services:
+ - docker
+
+install:
+ - pip install PyYAML
+ - pip install virtualenv
+ - |
+ test -e Gemfile || cat <<EOF > Gemfile
+ source 'https://rubygems.org'
+ gem 'rake'
+ gem 'test-kitchen'
+ gem 'kitchen-docker'
+ gem 'kitchen-vagrant'
+ gem 'kitchen-inspec'
+ gem 'inspec'
+ gem 'kitchen-salt', :git => 'https://github.com/salt-formulas/kitchen-salt.git'
+ - bundle install
+
+before_script:
+ - set -o pipefail
+ - make test | tail
+
+script:
+ - test ! -e .kitchen.yml || bundle exec kitchen converge $SUITE || true
+ - test ! -e .kitchen.yml || bundle exec kitchen verify $SUITE -t tests/integration
+
+notifications:
+ webhooks:
+ urls:
+ - https://webhooks.gitter.im/e/6123573504759330786b
+ on_success: change # options: [always|never|change] default: always
+ on_failure: never # options: [always|never|change] default: always
+ on_start: never # options: [always|never|change] default: always
+ on_cancel: never # options: [always|never|change] default: always
+ on_error: never # options: [always|never|change] default: always
+ email: false
diff --git a/kubernetes/_common.sls b/kubernetes/_common.sls
index f9a6902..ce62b38 100644
--- a/kubernetes/_common.sls
+++ b/kubernetes/_common.sls
@@ -33,22 +33,31 @@
- force: True
- require:
- file: /tmp/hyperkube
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
/usr/bin/hyperkube:
file.managed:
- - source: /tmp/hyperkube/hyperkube
- - mode: 751
- - makedirs: true
- - user: root
- - group: root
- - require:
- - dockerng: hyperkube-copy
+ - source: /tmp/hyperkube/hyperkube
+ - mode: 751
+ - makedirs: true
+ - user: root
+ - group: root
+ - require:
+ - dockerng: hyperkube-copy
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
/usr/bin/kubectl:
file.symlink:
- target: /usr/bin/hyperkube
- require:
- file: /usr/bin/hyperkube
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
/etc/systemd/system/kubelet.service:
file.managed:
@@ -61,12 +70,6 @@
/etc/kubernetes/config:
file.absent
-/etc/kubernetes/manifests:
- file.directory:
- - user: root
- - group: root
- - mode: 0751
-
{%- if not pillar.kubernetes.pool is defined %}
/etc/default/kubelet:
@@ -77,6 +80,15 @@
- group: root
- mode: 644
+/etc/kubernetes/kubelet.kubeconfig:
+ file.managed:
+ - source: salt://kubernetes/files/kubelet/kubelet.kubeconfig.master
+ - template: jinja
+ - user: root
+ - group: root
+ - mode: 644
+ - makedirs: true
+
{%- else %}
/etc/default/kubelet:
@@ -87,24 +99,25 @@
- group: root
- mode: 644
-{%- endif %}
-
-manifest_dir_create:
- file.directory:
- - name: /etc/kubernetes/manifests
- - user: root
- - group: root
- - mode: 0751
-
/etc/kubernetes/kubelet.kubeconfig:
file.managed:
- - source: salt://kubernetes/files/kubelet/kubelet.kubeconfig
+ - source: salt://kubernetes/files/kubelet/kubelet.kubeconfig.pool
- template: jinja
- user: root
- group: root
- mode: 644
- makedirs: true
+{%- endif %}
+
+manifest_dir_create:
+ file.directory:
+ - makedirs: true
+ - name: /etc/kubernetes/manifests
+ - user: root
+ - group: root
+ - mode: 0751
+
kubelet_service:
service.running:
- name: kubelet
@@ -114,6 +127,9 @@
- file: /usr/bin/hyperkube
- file: /etc/kubernetes/kubelet.kubeconfig
- file: manifest_dir_create
+ {% if grains.noservices is defined %}
+ - onlyif: {% if grains.get('noservices', "True") %}"True"{% else %}False{% endif %}
+ {% endif %}
{%- if common.logrotate is defined %}
/etc/logrotate.d/kubernetes:
diff --git a/kubernetes/files/calico/calicoctl.cfg.master b/kubernetes/files/calico/calicoctl.cfg.master
index 0a9fecd..d005e5a 100644
--- a/kubernetes/files/calico/calicoctl.cfg.master
+++ b/kubernetes/files/calico/calicoctl.cfg.master
@@ -4,8 +4,8 @@
metadata:
spec:
datastoreType: "etcdv2"
- etcdEndpoints: {% for member in master.network.etcd.members %}http{% if pool.network.etcd.get('ssl', {}).get('enabled') %}s{% endif %}://{{ member.host }}:{{ member.port }}{% if not loop.last %},{% endif %}{% endfor %}
-{%- if pool.network.etcd.get('ssl', {}).get('enabled') %}
+ etcdEndpoints: {% for member in master.network.etcd.members %}http{% if master.network.etcd.get('ssl', {}).get('enabled') %}s{% endif %}://{{ member.host }}:{{ member.port }}{% if not loop.last %},{% endif %}{% endfor %}
+{%- if master.network.etcd.get('ssl', {}).get('enabled') %}
etcdKeyFile: /var/lib/etcd/etcd-client.pem
etcdCertFile: /var/lib/etcd/etcd-client.pem
etcdCACertFile: /var/lib/etcd/ca.pem
diff --git a/kubernetes/files/kube-addons/calico-policy/calico-policy-controller.yml b/kubernetes/files/kube-addons/calico-policy/calico-policy-controller.yml
index b5f25e9..670506c 100644
--- a/kubernetes/files/kube-addons/calico-policy/calico-policy-controller.yml
+++ b/kubernetes/files/kube-addons/calico-policy/calico-policy-controller.yml
@@ -1,5 +1,4 @@
{%- from "kubernetes/map.jinja" import master with context %}
-{%- from "kubernetes/map.jinja" import pool with context %}
apiVersion: extensions/v1beta1
kind: ReplicaSet
metadata:
@@ -39,7 +38,7 @@
memory: 64M
env:
- name: ETCD_ENDPOINTS
- value: "{% for member in pool.network.etcd.members %}http{% if pool.network.etcd.get('ssl', {}).get('enabled') %}s{% endif %}://{{ member.host }}:{{ member.port }}{% if not loop.last %},{% endif %}{% endfor %}"
+ value: "{% for member in master.network.etcd.members %}http{% if master.network.etcd.get('ssl', {}).get('enabled') %}s{% endif %}://{{ member.host }}:{{ member.port }}{% if not loop.last %},{% endif %}{% endfor %}"
- name: ETCD_CA_CERT_FILE
value: "/var/lib/etcd/ca.pem"
- name: ETCD_CERT_FILE
diff --git a/kubernetes/files/kubelet/kubelet.kubeconfig.master b/kubernetes/files/kubelet/kubelet.kubeconfig.master
new file mode 100644
index 0000000..dd887f6
--- /dev/null
+++ b/kubernetes/files/kubelet/kubelet.kubeconfig.master
@@ -0,0 +1,21 @@
+{%- from "kubernetes/map.jinja" import master with context %}
+
+apiVersion: v1
+kind: Config
+current-context: cluster.local
+preferences: {}
+clusters:
+- cluster:
+ certificate-authority: /etc/kubernetes/ssl/ca-kubernetes.crt
+ server: https://{{ master.apiserver.address }}:443
+ name: cluster.local
+contexts:
+- context:
+ cluster: cluster.local
+ user: kubelet
+ name: cluster.local
+users:
+- name: kubelet
+ user:
+ client-certificate: /etc/kubernetes/ssl/kubelet-client.crt
+ client-key: /etc/kubernetes/ssl/kubelet-client.key
diff --git a/kubernetes/files/kubelet/kubelet.kubeconfig b/kubernetes/files/kubelet/kubelet.kubeconfig.pool
similarity index 100%
rename from kubernetes/files/kubelet/kubelet.kubeconfig
rename to kubernetes/files/kubelet/kubelet.kubeconfig.pool
diff --git a/kubernetes/files/manifest/kube-proxy.manifest.pool b/kubernetes/files/manifest/kube-proxy.manifest.pool
index 611a95b..2d563e9 100644
--- a/kubernetes/files/manifest/kube-proxy.manifest.pool
+++ b/kubernetes/files/manifest/kube-proxy.manifest.pool
@@ -17,7 +17,7 @@
- /hyperkube
- proxy
--logtostderr=true
- --v={{ master.get('verbosity', 2) }}
+ --v={{ pool.get('verbosity', 2) }}
--kubeconfig=/etc/kubernetes/proxy.kubeconfig
--master={%- if pool.apiserver.insecure.enabled %}http://{{ pool.apiserver.host }}:8080{%- else %}https://{{ pool.apiserver.host }}{%- endif %}
{%- if pool.network.engine == 'calico' %}
diff --git a/kubernetes/master/calico.sls b/kubernetes/master/calico.sls
index 8ecc181..7f58c62 100644
--- a/kubernetes/master/calico.sls
+++ b/kubernetes/master/calico.sls
@@ -33,7 +33,7 @@
/etc/systemd/system/calico-node.service:
file.managed:
- - source: salt://kubernetes/files/calico/calico-node.service.pool.master
+ - source: salt://kubernetes/files/calico/calico-node.service.master
- user: root
- group: root
- template: jinja
@@ -44,7 +44,10 @@
- enable: True
- watch:
- file: /etc/systemd/system/calico-node.service
-
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+
{%- endif %}
{%- endif %}
diff --git a/kubernetes/master/controller.sls b/kubernetes/master/controller.sls
index 80c4e53..72f71fb 100644
--- a/kubernetes/master/controller.sls
+++ b/kubernetes/master/controller.sls
@@ -216,6 +216,9 @@
cmd.run:
- name: kubectl create ns "{{ name }}"
- name: kubectl get ns -o=custom-columns=NAME:.metadata.name | grep -v NAME | grep "{{ name }}" > /dev/null || kubectl create ns "{{ name }}"
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
{%- else %}
@@ -232,12 +235,19 @@
cmd.run:
- name: bash -c 'while ! kubectl get nodes {{ master.host.name }}; do sleep 5; done'
- timeout: 180
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
kubernetes_taint_master_{{ master.host.name }}:
cmd.run:
- name: kubectl taint --overwrite nodes {{ master.host.name }} node-role.kubernetes.io/master=:NoSchedule
- require:
- cmd: kubernetes_node_ready_{{ master.host.name}}
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+
{%- endif %}
{%- if master.registry.secret is defined %}
@@ -249,6 +259,9 @@
/registry/secrets/{{ registry.namespace }}/{{ name }}:
etcd.set:
- value: '{"kind":"Secret","apiVersion":"v1","metadata":{"name":"{{ name }}","namespace":"{{ registry.namespace }}"},"data":{".dockerconfigjson":"{{ registry.key }}"},"type":"kubernetes.io/dockerconfigjson"}'
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
{%- else %}
diff --git a/kubernetes/master/setup.sls b/kubernetes/master/setup.sls
index 694b4b2..f27035b 100644
--- a/kubernetes/master/setup.sls
+++ b/kubernetes/master/setup.sls
@@ -9,7 +9,10 @@
- name: |
hyperkube kubectl apply -f /etc/kubernetes/addons/{{ addon_name }}
- unless: "hyperkube kubectl get svc kube-{{ addon.get('name', addon_name) }} --namespace={{ addon.get('namespace', 'kube-system') }}"
-
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+
{%- endif %}
{%- endfor %}
@@ -25,6 +28,9 @@
- value: {{ label.value }}
- node: {{ label.node }}
- apiserver: http://{{ master.apiserver.insecure_address }}:{{ master.apiserver.get('insecure_port', '8080') }}
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
{%- else %}
@@ -33,6 +39,9 @@
- name: {{ label.key }}
- node: {{ label.node }}
- apiserver: http://{{ master.apiserver.insecure_address }}:{{ master.apiserver.get('insecure_port', '8080') }}
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
{%- endif %}
diff --git a/kubernetes/pool/calico.sls b/kubernetes/pool/calico.sls
index e38ece0..9e31551 100644
--- a/kubernetes/pool/calico.sls
+++ b/kubernetes/pool/calico.sls
@@ -9,40 +9,58 @@
copy-calico-ctl:
dockerng.running:
- image: {{ pool.network.calicoctl.image }}
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
copy-calico-ctl-cmd:
cmd.run:
- name: docker cp copy-calico-ctl:calicoctl /tmp/calico/
- require:
- dockerng: copy-calico-ctl
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
/usr/bin/calicoctl:
file.managed:
- - source: /tmp/calico/calicoctl
- - mode: 751
- - user: root
- - group: root
- - require:
- - cmd: copy-calico-ctl-cmd
+ - source: /tmp/calico/calicoctl
+ - mode: 751
+ - user: root
+ - group: root
+ - require:
+ - cmd: copy-calico-ctl-cmd
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
copy-calico-node:
dockerng.running:
- image: {{ pool.network.get('image', 'calico/node') }}
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
copy-bird-cl-cmd:
cmd.run:
- name: docker cp copy-calico-node:/bin/birdcl /tmp/calico/
- require:
- dockerng: copy-calico-node
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
/usr/bin/birdcl:
file.managed:
- - source: /tmp/calico/birdcl
- - mode: 751
- - user: root
- - group: root
- - require:
- - cmd: copy-bird-cl-cmd
+ - source: /tmp/calico/birdcl
+ - mode: 751
+ - user: root
+ - group: root
+ - require:
+ - cmd: copy-bird-cl-cmd
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
copy-calico-cni:
dockerng.running:
@@ -51,20 +69,27 @@
- binds:
- /tmp/calico/:/tmp/calico/
- force: True
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
{%- for filename in ['calico', 'calico-ipam'] %}
/opt/cni/bin/{{ filename }}:
file.managed:
- - source: /tmp/calico/bin/{{ filename }}
- - mode: 751
- - makedirs: true
- - user: root
- - group: root
- - require:
- - dockerng: copy-calico-cni
- - require_in:
- - service: calico_node
+ - source: /tmp/calico/bin/{{ filename }}
+ - mode: 751
+ - makedirs: true
+ - user: root
+ - group: root
+ - require:
+ - dockerng: copy-calico-cni
+ - require_in:
+ - service: calico_node
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+
{%- endfor %}
/etc/cni/net.d/10-calico.conf:
@@ -112,6 +137,10 @@
- enable: True
- watch:
- file: /etc/systemd/system/calico-node.service
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+
{%- endif %}
{%- endif %}
diff --git a/kubernetes/pool/cni.sls b/kubernetes/pool/cni.sls
index bf7f2bf..ffcf2ad 100644
--- a/kubernetes/pool/cni.sls
+++ b/kubernetes/pool/cni.sls
@@ -18,6 +18,9 @@
- force: True
- require:
- file: /tmp/cni/
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
{%- for filename in ['cnitool', 'flannel', 'tuning', 'bridge', 'ipvlan', 'loopback', 'macvlan', 'ptp', 'dhcp', 'host-local', 'noop'] %}
/opt/cni/bin/{{ filename }}:
@@ -31,6 +34,9 @@
- service: kubelet_service
- require:
- dockerng: copy-network-cni
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
{%- endfor %}
diff --git a/kubernetes/pool/kube-proxy.sls b/kubernetes/pool/kube-proxy.sls
index b065f4f..cf0ad12 100644
--- a/kubernetes/pool/kube-proxy.sls
+++ b/kubernetes/pool/kube-proxy.sls
@@ -47,6 +47,9 @@
- file: /etc/default/kube-proxy
- file: /usr/bin/hyperkube
- file: /etc/kubernetes/proxy.kubeconfig
+ {% if grains.noservices is defined %}
+ - onlyif: {% if grains.get('noservices', "True") %}"True"{% else %}False{% endif %}
+ {% endif %}
{%- endif %}
diff --git a/tests/pillar/master_cluster.sls b/tests/pillar/master_cluster.sls
index 86f0193..dd65e6e 100644
--- a/tests/pillar/master_cluster.sls
+++ b/tests/pillar/master_cluster.sls
@@ -8,7 +8,7 @@
addons:
dns:
domain: cluster.local
- enabled: true
+ enabled: false
replicas: 1
server: 10.254.0.10
autoscaler:
@@ -24,8 +24,15 @@
tiller_image: gcr.io/kubernetes-helm/tiller:v2.2.3
netchecker:
enabled: true
+ namespace: netchecker
+ port: 80
+ interval: 60
+ server_image: image
+ agent_image: image
calico_policy:
enabled: true
+ namespace: kube-system
+ image: image
admin:
password: password
username: admin
@@ -51,6 +58,14 @@
hash: fb5e30ebe6154911a66ec3fb5f1195b2
private_ip_range: 10.150.0.0/16
version: v0.19.0
+ etcd:
+ members:
+ - host: 127.0.0.1
+ port: 4001
+ - host: 127.0.0.1
+ port: 4001
+ - host: 127.0.0.1
+ port: 4001
service_addresses: 10.254.0.0/16
storage:
engine: glusterfs
@@ -81,7 +96,7 @@
hash: hnsj0XqABgrSww7Nqo7UVTSZLJUt2XRd
services:
myservice:
- enabled: true
+ enabled: false
files:
- /srv/kubernetes/myservice-svc.yml
- /srv/kubernetes/myservice-pvc.yml
diff --git a/tests/pillar/repo_docker.sls b/tests/pillar/repo_docker.sls
new file mode 100644
index 0000000..098c177
--- /dev/null
+++ b/tests/pillar/repo_docker.sls
@@ -0,0 +1,12 @@
+linux:
+ system:
+ enabled: true
+ repo:
+ docker:
+ source: "deb https://apt.dockerproject.org/repo ubuntu-{{ grains.get('oscodename') }} main"
+ architectures: amd64
+ key_id: 58118E89F3A912897C070ADBF76221572C52609D
+ key_server: hkp://p80.pool.sks-keyservers.net:80
+ package:
+ python-docker:
+ version: latest