diff --git a/.kitchen.yml b/.kitchen.yml
index 74272ff..5a065c2 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -51,16 +51,10 @@
   sudo: true
 
 docker_images:
-  - &xenial-20163 <%=ENV['IMAGE_XENIAL_20163'] || 'docker-dev-local.docker.mirantis.net/epcim/salt/saltstack-ubuntu-xenial-salt-2016.3/salt:2018_11_19'%>
   - &xenial-20177 <%=ENV['IMAGE_XENIAL_20177'] || 'docker-dev-local.docker.mirantis.net/epcim/salt/saltstack-ubuntu-xenial-salt-2017.7/salt:2018_11_19'%>
   - &xenial-stable <%=ENV['IMAGE_XENIAL_STABLE'] || 'docker-dev-local.docker.mirantis.net/epcim/salt/saltstack-ubuntu-xenial-salt-stable/salt:2018_11_19'%>
 
 platforms:
-  - name: xenial-2016.3
-    driver_config:
-      image: *xenial-20163
-      platform: ubuntu
-
   - name: xenial-2017.7
     driver_config:
       image: *xenial-20177
diff --git a/README.rst b/README.rst
index 263d1a3..3cf026d 100644
--- a/README.rst
+++ b/README.rst
@@ -1185,6 +1185,29 @@
                   test:
                     kind: User
 
+Manage client (kubectl resources)
+Assign labels to nodes:
+
+.. code-block:: yaml
+
+    kubernetes:
+      client:
+        enabled: true
+        apiserver:
+          insecure_address: 0.0.0.0
+          insecure_port: 8080
+        resources:
+          enabled: true
+          label:
+            test:
+              value: enabled
+              status: 'present'
+              node:
+               - cmp1
+               - cmp2
+              enabled: true
+              key: mylabel
+
 More Information
 ================
 
diff --git a/kubernetes/_common.sls b/kubernetes/_common.sls
index b74a76a..3d59fbf 100644
--- a/kubernetes/_common.sls
+++ b/kubernetes/_common.sls
@@ -36,6 +36,7 @@
 /etc/containerd/config.toml:
   file.managed:
   - source: salt://kubernetes/files/containerd/config.toml
+  - makedirs: True
   - template: jinja
   - user: root
   - group: root
@@ -134,6 +135,12 @@
     - group: root
     - mode: 0750
 
+/var/log/criproxy:
+  file.directory:
+    - user: root
+    - group: root
+    - mode: 0750
+
 /etc/criproxy/node.conf:
   file.managed:
     - user: root
@@ -192,6 +199,7 @@
   file.absent
 
 {%- if common.get('cloudprovider', {}).get('enabled') and common.get('cloudprovider', {}).get('provider') == 'openstack' %}
+{%- set cloudconfig_type = 'external' %}
 /etc/kubernetes/cloud-config:
   file.managed:
   - source: salt://kubernetes/files/cloudprovider/cloud-config-openstack.conf
@@ -199,6 +207,21 @@
   - user: root
   - group: root
   - mode: 600
+  - defaults:
+      cloudconfig_type: {{ cloudconfig_type }}
+
+{%- if pillar.kubernetes.master is defined %}
+{%- set cloudconfig_type = 'intree' %}
+/etc/kubernetes/cloud-config.intree:
+  file.managed:
+  - source: salt://kubernetes/files/cloudprovider/cloud-config-openstack.conf
+  - template: jinja
+  - user: root
+  - group: root
+  - mode: 600
+  - defaults:
+      cloudconfig_type: {{ cloudconfig_type }}
+{% endif %}
 
 {% endif %}
 
@@ -294,7 +317,7 @@
     - options: xzf
     {%- endif %}
     - archive_format: tar
-    - if_missing: /opt/cni/bin/host-local
+    - overwrite: true
 {%- endif %}
 
 {%- if common.addons.get('helm', {'enabled': False}).enabled %}
diff --git a/kubernetes/client.sls b/kubernetes/client.sls
new file mode 100644
index 0000000..9c1c383
--- /dev/null
+++ b/kubernetes/client.sls
@@ -0,0 +1,33 @@
+{%- from "kubernetes/map.jinja" import client with context -%}
+{%- if client.enabled %}
+  {%- if client.get('resources', {}).get('enabled') %}
+
+    {%- for name,label in client.resources.get('label', {}).iteritems() %}
+
+      {%- if label.enabled %}
+        {%- if label.get('status', 'present') == 'present' %}
+          {%- for node in label.node %}
+# TODO(vsaienko) switch to kubernetes. salt module once kubernets-client python is packages and
+# awailable for installation.
+{{ name }}_{{ node }}:
+  k8s.label_present:
+    - name: {{ label.key }}
+    - value: {{ label.value }}
+    - node: {{ node }}
+    # TODO(vsaienko): move to profiles
+    - apiserver: http://{{ client.apiserver.insecure_address }}:{{ client.apiserver.insecure_port }}
+          {%- endfor %}
+
+        {%- elif label.get('status', 'present') == 'absent' %}
+          {%- for node in label.node %}
+{{ name }}_{{ node }}:
+  k8s.label_absent:
+    - name: {{ label.key }}
+    - node: {{ node }}
+    - apiserver: http://{{ client.apiserver.insecure_address }}:{{ client.apiserver.insecure_port }}
+          {%- endfor %} # endfor label.node.iteritems
+        {%- endif %} # endif label.present
+      {%- endif %} # endif label.enabled
+    {%- endfor %} # endfor client.resources.label
+  {%- endif %} # endif client.resources.enabled
+{%- endif %} # endif client.enabled
diff --git a/kubernetes/control/endpoint.sls b/kubernetes/control/endpoint.sls
new file mode 100644
index 0000000..962855e
--- /dev/null
+++ b/kubernetes/control/endpoint.sls
@@ -0,0 +1,60 @@
+{% from "kubernetes/map.jinja" import control with context %}
+include:
+  - kubernetes.control
+
+{%- for endpoint_name, endpoint in control.endpoints.items() %}
+  {%- if endpoint.get('service_enabled', false) %}
+
+/srv/kubernetes/services/{{ endpoint.cluster }}/{{ endpoint.service }}-svc.yml:
+  file.managed:
+  - source: salt://kubernetes/files/svc.yml
+  - user: root
+  - group: root
+  - template: jinja
+  - makedirs: true
+  - require:
+    - file: /srv/kubernetes
+  - defaults:
+      service: {{ endpoint|yaml }}
+
+    {%- if endpoint.get('create', false) %}
+      {%- set service_name = endpoint.service + '-' + endpoint.role if endpoint.role is defined else endpoint.service %}
+kubernetes_service_create_{{ endpoint.service }}:
+  cmd.wait:
+    - name: kubectl apply -f /srv/kubernetes/services/{{ endpoint.cluster }}/{{ endpoint.service }}-svc.yml
+    - unless: kubectl get service -o=custom-columns=NAME:.metadata.name --namespace {{ endpoint.namespace }} | grep -xq {{ endpoint.service }}
+    {%- if grains.get('noservices') %}
+    - onlyif: /bin/false
+    {%- endif %}
+    - watch:
+      - file: /srv/kubernetes/services/{{ endpoint.cluster }}/{{ endpoint.service }}-svc.yml
+    {%- endif %}
+
+  {%- endif %}
+
+/srv/kubernetes/endpoints/{{ endpoint.cluster }}/{{ endpoint_name }}.yml:
+  file.managed:
+  - source: salt://kubernetes/files/endpoint.yml
+  - user: root
+  - group: root
+  - template: jinja
+  - makedirs: true
+  - require:
+    - file: /srv/kubernetes
+  - defaults:
+      endpoint: {{ endpoint|yaml }}
+      endpoint_name: {{ endpoint_name }}
+
+    {%- if endpoint.get('create', false) %}
+kubernetes_endpoint_create_{{ endpoint_name }}:
+  cmd.wait:
+    - name: kubectl apply -f /srv/kubernetes/endpoints/{{ endpoint.cluster }}/{{ endpoint_name }}.yml
+    - unless: kubectl get endpoint -o=custom-columns=NAME:.metadata.name --namespace {{ endpoint.namespace }} | grep -xq {{ endpoint_name }}
+    {%- if grains.get('noservices') %}
+    - onlyif: /bin/false
+    {%- endif %}
+    - watch:
+      - file: /srv/kubernetes/endpoints/{{ endpoint.cluster }}/{{ endpoint_name }}.yml
+    {%- endif %}
+
+{%- endfor %}
diff --git a/kubernetes/control/ingress.sls b/kubernetes/control/ingress.sls
new file mode 100644
index 0000000..8a262dc
--- /dev/null
+++ b/kubernetes/control/ingress.sls
@@ -0,0 +1,34 @@
+{% from "kubernetes/map.jinja" import control with context %}
+include:
+  - kubernetes.control
+
+{%- for ingress_name, ingress in control.ingress.items() %}
+  {%- if ingress.get('enabled', false) %}
+
+/srv/kubernetes/ingress/{{ ingress.cluster }}/{{ ingress_name }}-ingress.yml:
+  file.managed:
+  - source: salt://kubernetes/files/ingress.yml
+  - user: root
+  - group: root
+  - template: jinja
+  - makedirs: true
+  - require:
+    - file: /srv/kubernetes
+  - defaults:
+      ingress: {{ ingress|yaml }}
+      ingress_name: {{ ingress_name }}
+
+    {%- if ingress.get('create', false) %}
+kubernetes_ingress_create_{{ ingress_name }}:
+  cmd.wait:
+    - name: kubectl apply -f /srv/kubernetes/ingress/{{ ingress.cluster }}/{{ ingress_name }}-ingress.yml
+    - unless: kubectl get ingress -o=custom-columns=NAME:.metadata.name --namespace {{ ingress.namespace }} | grep -xq {{ ingress_name }}
+    {%- if grains.get('noservices') %}
+    - onlyif: /bin/false
+    {%- endif %}
+    - watch:
+      - file: /srv/kubernetes/ingress/{{ ingress.cluster }}/{{ ingress_name }}-ingress.yml
+    {%- endif %}
+
+  {%- endif %}
+{%- endfor %}
diff --git a/kubernetes/control/init.sls b/kubernetes/control/init.sls
index b8ea755..18d8b11 100644
--- a/kubernetes/control/init.sls
+++ b/kubernetes/control/init.sls
@@ -15,6 +15,12 @@
   {%- if control.priorityclass is defined %}
   - kubernetes.control.priorityclass
   {%- endif %}
+  {%- if control.endpoints is defined %}
+  - kubernetes.control.endpoint
+  {%- endif %}
+  {%- if control.ingress is defined %}
+  - kubernetes.control.ingress
+  {%- endif %}
 
 /srv/kubernetes:
   file.directory:
diff --git a/kubernetes/control/service.sls b/kubernetes/control/service.sls
index e7e9330..4e94a56 100644
--- a/kubernetes/control/service.sls
+++ b/kubernetes/control/service.sls
@@ -3,7 +3,7 @@
   - kubernetes.control
 
 {%- for service_name, service in control.service.items() %}
-  {%- if service.enabled %}
+  {%- if service.get('enabled', false) %}
 
 /srv/kubernetes/services/{{ service.cluster }}/{{ service_name }}-svc.yml:
   file.managed:
@@ -17,6 +17,19 @@
   - defaults:
       service: {{ service|yaml }}
 
+    {%- if service.get('create', false) %}
+      {%- set service_real_name = service.service + '-' + service.role if service.role is defined else service.service %}
+kubernetes_service_create_{{ service_name }}:
+  cmd.wait:
+    - name: kubectl apply -f /srv/kubernetes/services/{{ service.cluster }}/{{ service_name }}-svc.yml
+    - unless: kubectl get service -o=custom-columns=NAME:.metadata.name --namespace {{ service.namespace }} | grep -xq {{ service_real_name }}
+    {%- if grains.get('noservices') %}
+    - onlyif: /bin/false
+    {%- endif %}
+    - watch:
+      - file: /srv/kubernetes/services/{{ service.cluster }}/{{ service_name }}-svc.yml
+    {%- endif %}
+
   {%- endif %}
 
 /srv/kubernetes/{{ service.kind|lower }}/{{ service_name }}-{{ service.kind }}.yml:
@@ -31,6 +44,19 @@
   - defaults:
       service: {{ service|yaml }}
 
+    {%- if service.get('create', false) %}
+      {%- set service_real_name = service.service + '-' + service.role if service.role is defined else service.service %}
+kubernetes_{{ service.kind|lower }}_create_{{ service_name }}:
+  cmd.wait:
+    - name: kubectl apply -f /srv/kubernetes/{{ service.kind|lower }}/{{ service_name }}-{{ service.kind }}.yml
+    - unless: kubectl get {{ service.kind|lower }} -o=custom-columns=NAME:.metadata.name --namespace {{ service.namespace }} | grep -xq {{ service_real_name }}
+    {%- if grains.get('noservices') %}
+    - onlyif: /bin/false
+    {%- endif %}
+    - watch:
+      - file: /srv/kubernetes/{{ service.kind|lower }}/{{ service_name }}-{{ service.kind }}.yml
+    {%- endif %}
+
 {%- endfor %}
 
 {%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').items() %}
@@ -51,6 +77,19 @@
   - defaults:
       service: {{ service|yaml }}
 
+      {%- if service.get('create', false) %}
+        {%- set service_real_name = service.service + '-' + service.role if service.role is defined else service.service %}
+kubernetes_service_create_{{ service.service }}:
+  cmd.wait:
+    - name: kubectl apply -f /srv/kubernetes/services/{{ node_name }}-svc.yml
+    - unless: kubectl get service -o=custom-columns=NAME:.metadata.name --namespace {{ service.namespace }} | grep -xq {{ service_real_name }}
+    {%- if grains.get('noservices') %}
+    - onlyif: /bin/false
+    {%- endif %}
+    - watch:
+      - file: /srv/kubernetes/services/{{ node_name }}-svc.yml
+      {%- endif %}
+
     {%- endif %}
 /srv/kubernetes/{{ service.kind|lower }}/{{ node_name }}-{{ service.kind }}.yml:
   file.managed:
@@ -64,6 +103,19 @@
   - defaults:
       service: {{ service|yaml }}
 
+    {%- if service.get('create', false) %}
+      {%- set service_real_name = service.service + '-' + service.role if service.role is defined else service.service %}
+kubernetes_{{ service.kind|lower }}_create_{{ service_name }}:
+  cmd.wait:
+    - name: kubectl apply -f /srv/kubernetes/{{ service.kind|lower }}/{{ node_name }}-{{ service.kind }}.yml
+    - unless: kubectl get {{ service.kind|lower }} -o=custom-columns=NAME:.metadata.name --namespace {{ service.namespace }} | grep -xq {{ service_real_name }}
+    {%- if grains.get('noservices') %}
+    - onlyif: /bin/false
+    {%- endif %}
+    - watch:
+      - file: /srv/kubernetes/{{ service.kind|lower }}/{{ node_name }}-{{ service.kind }}.yml
+    {%- endif %}
+
   {%- endif %}
 
 {%- endfor %}
diff --git a/kubernetes/files/cloudprovider/cloud-config-openstack.conf b/kubernetes/files/cloudprovider/cloud-config-openstack.conf
index 92a1cdd..83bdd8d 100644
--- a/kubernetes/files/cloudprovider/cloud-config-openstack.conf
+++ b/kubernetes/files/cloudprovider/cloud-config-openstack.conf
@@ -22,15 +22,15 @@
 
 
 [LoadBalancer]
-{%- if common.cloudprovider.params.subnet_id is defined %}
 use-octavia=true
+{%- if common.cloudprovider.params.subnet_id is defined %}
 subnet-id={{ common.cloudprovider.params.subnet_id }}
 {%- endif %}
 {%- if common.cloudprovider.params.lb_method is defined %}
 lb-method={{ common.cloudprovider.params.lb_method }}
 {%- endif %}
-{%- if common.cloudprovider.params.floating_network_id is defined %}
-floating-network-id={{ common.cloudprovider.params.floating_network_id }}
+{%- if common.cloudprovider.params.floating_net_id is defined %}
+floating-network-id={{ common.cloudprovider.params.floating_net_id }}
 {%- endif %}
 {%- if common.cloudprovider.params.create_monitor is defined %}
 create-monitor={{ common.cloudprovider.params.create_monitor }}
@@ -46,3 +46,17 @@
 
 [BlockStorage]
 ignore-volume-az=true
+
+
+{%- if pillar.kubernetes.master is defined and cloudconfig_type == "external" %}
+[Networking]
+{%- if common.cloudprovider.params.internal_net_name is defined %}
+internal-network-name={{ common.cloudprovider.params.internal_net_name }}
+{%- endif %}
+{%- if common.cloudprovider.params.public_net_name is defined %}
+public-network-name={{ common.cloudprovider.params.public_net_name }}
+{%- endif %}
+{%- if common.cloudprovider.params.ipv6_support_disabled is defined %}
+ipv6-support-disabled={{ common.cloudprovider.params.ipv6_support_disabled }}
+{%- endif %}
+{%- endif %}
diff --git a/kubernetes/files/endpoint.yml b/kubernetes/files/endpoint.yml
new file mode 100644
index 0000000..6109bf8
--- /dev/null
+++ b/kubernetes/files/endpoint.yml
@@ -0,0 +1,18 @@
+{% from "kubernetes/map.jinja" import control with context %}
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: {{ endpoint_name }}
+  {%- if endpoint.namespace is defined %}
+  namespace: {{ endpoint.namespace }}
+  {%- endif %}
+subsets:
+{%- for subset in endpoint.subsets %}
+  - addresses:
+      - ip: {{ subset.ip }}
+    ports:
+      - port: {{ subset.port.number }}
+      {%- if subset.port.name is defined %}
+        name: {{ subset.port.name }}
+      {%- endif %}
+{%- endfor %}
diff --git a/kubernetes/files/ingress.yml b/kubernetes/files/ingress.yml
new file mode 100644
index 0000000..5fb1cf0
--- /dev/null
+++ b/kubernetes/files/ingress.yml
@@ -0,0 +1,41 @@
+{% from "kubernetes/map.jinja" import control with context %}
+apiVersion: {{ ingress.apiVersion }}
+kind: Ingress
+metadata:
+  name: {{ ingress_name }}
+  namespace: {{ ingress.namespace }}
+  {%- if ingress.annotations is defined %}
+  annotations:
+    {%- for annotation in ingress.annotations %}
+    {{ annotation.name }}: "{{ annotation.value }}"
+    {%- endfor %}
+  {%- endif %}
+spec:
+  {%- if ingress.tls is defined %}
+  tls:
+  - hosts:
+    {%- for host in ingress.tls.hosts %}
+    - {{ host }}
+    {%- endfor %}
+    secretName: {{ ingress.tls.secret_name }}
+  {%- endif %}
+  {%- if ingress.rules is defined %}
+  rules:
+    {%- for host in ingress.rules.hosts %}
+  - {%- if host.name is defined %}
+    host: {{ host.name }}
+    {%- endif %}
+    http:
+      paths:
+      {%- for path, backend in host.paths.items() %}
+      - path: {{ path }}
+        backend:
+          serviceName: {{ backend.service }}
+          servicePort: {{ backend.port }}
+      {%- endfor %}
+    {%- endfor %}
+  {%- elif ingress.backend is defined %}
+  backend:
+    serviceName: {{ ingress.backend.service }}
+    servicePort: {{ ingress.backend.port }}
+  {%- endif %}
diff --git a/kubernetes/files/kube-addons/contrail/contrail.yaml b/kubernetes/files/kube-addons/contrail/contrail.yaml
index 2adef76..ee56429 100644
--- a/kubernetes/files/kube-addons/contrail/contrail.yaml
+++ b/kubernetes/files/kube-addons/contrail/contrail.yaml
@@ -1,5 +1,14 @@
 {%- from "kubernetes/map.jinja" import common with context -%}
 {%- from "kubernetes/map.jinja" import master with context -%}
+
+{%- if  master.network.get('opencontrail',{}).get('version', 4.0)  >= 4.1 %}
+  {%- set kafka_path = "/etc/kafka" %}
+  {%- set kafka_path_log = "/var/log/kafka" %}
+
+{%- else %}
+  {%- set kafka_path = "/usr/share/kafka/config" %}
+  {%- set kafka_path_log = "/usr/share/kafka/logs" %}
+{%- endif %}
 ---
 
 apiVersion: apps/v1beta2
@@ -104,6 +113,8 @@
           mountPath: /var/log/contrail
         - name: journal-controller
           mountPath: /var/log/journal
+        - name: var-log-configdb-zookeeper
+          mountPath: /var/log/zookeeper
         - name: etc-hostname
           mountPath: /etc/hostname
           readOnly: true
@@ -132,14 +143,14 @@
           mountPath: /var/lib/zookeeper
         - name: var-lib-zookeeper-myid
           mountPath: /var/lib/zookeeper/myid
-        - name: usr-share-kafka-config-server-properties
-          mountPath: /usr/share/kafka/config/server.properties
-        - name: usr-share-kafka-config-consumer-properties
-          mountPath: /usr/share/kafka/config/consumer.properties
-        - name: usr-share-kafka-config-zookeeper-properties
-          mountPath: /usr/share/kafka/config/zookeeper.properties
-        - name: usr-share-kafka-logs
-          mountPath: /usr/share/kafka/logs
+        - name: kafka-config-server-properties
+          mountPath: {{ kafka_path }}/server.properties
+        - name: kafka-config-consumer-properties
+          mountPath: {{ kafka_path }}/consumer.properties
+        - name: kafka-config-zookeeper-properties
+          mountPath: {{ kafka_path }}/zookeeper.properties
+        - name: kafka-logs
+          mountPath: {{ kafka_path_log }}
         - name: etc-zookeeper-conf-zoo-analytics-cfg
           mountPath: /etc/zookeeper/conf/zoo.cfg
         - name: etc-zookeeper-conf-log4j-properties
@@ -148,6 +159,8 @@
           mountPath: /var/log/contrail
         - name: journal-analyticsdb
           mountPath: /var/log/journal
+        - name: var-log-analyticsdb-zookeeper
+          mountPath: /var/log/zookeeper
         - name: etc-hostname
           mountPath: /etc/hostname
           readOnly: true
@@ -215,6 +228,10 @@
         hostPath:
           path:  /var/log/journal/contrail-controller
           type: DirectoryOrCreate
+      - name: var-log-configdb-zookeeper
+        hostPath:
+          path: /var/log/configdb/zookeeper
+          type: DirectoryOrCreate
 
       # analyticsdb
       - name: etc-cassandra-cassandra-env-analytics-sh
@@ -241,21 +258,21 @@
         hostPath:
           path: /var/lib/zookeeper/myid
           type: File
-      - name: usr-share-kafka-config-server-properties
+      - name: kafka-config-server-properties
         hostPath:
-          path: /usr/share/kafka/config/server.properties
+          path: {{ kafka_path }}/server.properties
           type: File
-      - name: usr-share-kafka-config-consumer-properties
+      - name: kafka-config-consumer-properties
         hostPath:
-          path: /usr/share/kafka/config/consumer.properties
+          path: {{ kafka_path }}/consumer.properties
           type: File
-      - name: usr-share-kafka-config-zookeeper-properties
+      - name: kafka-config-zookeeper-properties
         hostPath:
-          path: /usr/share/kafka/config/zookeeper.properties
+          path: {{ kafka_path }}/zookeeper.properties
           type: File
-      - name: usr-share-kafka-logs
+      - name: kafka-logs
         hostPath:
-          path: /usr/share/kafka/logs
+          path: {{ kafka_path_log }}
           type: DirectoryOrCreate
       - name: etc-zookeeper-conf-zoo-analytics-cfg
         hostPath:
@@ -269,6 +286,10 @@
         hostPath:
           path:  /var/log/journal/contrail-analyticsdb
           type: DirectoryOrCreate
+      - name: var-log-analyticsdb-zookeeper
+        hostPath:
+          path: /var/log/analyticsdb/zookeeper
+          type: DirectoryOrCreate
 
       # analytics
       - name: etc-redis-redis-conf
diff --git a/kubernetes/files/kube-addons/ingress-nginx/ingress-nginx.yaml b/kubernetes/files/kube-addons/ingress-nginx/ingress-nginx.yaml
index aedb8be..7e28da8 100644
--- a/kubernetes/files/kube-addons/ingress-nginx/ingress-nginx.yaml
+++ b/kubernetes/files/kube-addons/ingress-nginx/ingress-nginx.yaml
@@ -86,6 +86,12 @@
 
 kind: ConfigMap
 apiVersion: v1
+{%- if common.addons.get('ingress-nginx', {}).tcp_data is defined %}
+data:
+  {%- for key, value in common.addons.get('ingress-nginx').tcp_data.items() %}
+    {{ key }}: "{{ value }}"
+  {%- endfor %}
+{%- endif %}
 metadata:
   name: tcp-services
   namespace: ingress-nginx
@@ -97,6 +103,12 @@
 
 kind: ConfigMap
 apiVersion: v1
+{%- if common.addons.get('ingress-nginx', {}).udp_data is defined %}
+data:
+  {%- for key, value in common.addons.get('ingress-nginx').udp_data.items() %}
+    {{ key }}: "{{ value }}"
+  {%- endfor %}
+{%- endif %}
 metadata:
   name: udp-services
   namespace: ingress-nginx
@@ -320,6 +332,18 @@
             containerPort: 80
           - name: https
             containerPort: 443
+          {%- if common.addons.get('ingress-nginx', {}).tcp_ports is defined %}
+            {%- for port in common.addons.get('ingress-nginx').tcp_ports %}
+          - name: tcp{{ port }}
+            containerPort: {{ port }}
+            {%- endfor %}
+          {%- endif %}
+          {%- if common.addons.get('ingress-nginx', {}).udp_ports is defined %}
+            {%- for port in common.addons.get('ingress-nginx').udp_ports %}
+          - name: udp{{ port }}
+            containerPort: {{ port }}
+            {%- endfor %}
+          {%- endif %}
           livenessProbe:
             failureThreshold: 3
             httpGet:
@@ -352,7 +376,7 @@
     app.kubernetes.io/part-of: ingress-nginx
     addonmanager.kubernetes.io/mode: Reconcile
 spec:
-  externalTrafficPolicy: Local
+  externalTrafficPolicy: {{ common.addons.get('ingress-nginx', {}).get('externalTrafficPolicy', 'Local') }}
   type: LoadBalancer
   selector:
     app.kubernetes.io/name: ingress-nginx
@@ -364,5 +388,18 @@
     - name: https
       port: 443
       targetPort: https
-
+    {%- if common.addons.get('ingress-nginx', {}).tcp_ports is defined %}
+      {%- for port in common.addons.get('ingress-nginx').tcp_ports %}
+    - name: tcp{{ port }}
+      port: {{ port }}
+      targetPort: tcp{{ port }}
+      {%- endfor %}
+    {%- endif %}
+    {%- if common.addons.get('ingress-nginx', {}).udp_ports is defined %}
+      {%- for port in common.addons.get('ingress-nginx').udp_ports %}
+    - name: udp{{ port }}
+      port: {{ port }}
+      targetPort: udp{{ port }}
+      {%- endfor %}`
+    {%- endif %}
 ---
diff --git a/kubernetes/files/kubelet/default.pool b/kubernetes/files/kubelet/default.pool
index 8e2a1e9..4ba5328 100644
--- a/kubernetes/files/kubelet/default.pool
+++ b/kubernetes/files/kubelet/default.pool
@@ -56,7 +56,7 @@
 {%- endif %}
 --file-check-frequency={{ pool.kubelet.frequency }} \
 {%- if common.get('cloudprovider', {}).get('enabled') and common.get('cloudprovider', {}).get('provider') == 'openstack' %}
---cloud-provider=openstack \
+--cloud-provider=external \
 --cloud-config=/etc/kubernetes/cloud-config \
 {%- endif %}
 {%- if common.addons.get('virtlet', {}).get('enabled') %}
diff --git a/kubernetes/files/opencontrail/4.1 b/kubernetes/files/opencontrail/4.1
new file mode 120000
index 0000000..0c16b90
--- /dev/null
+++ b/kubernetes/files/opencontrail/4.1
@@ -0,0 +1 @@
+4.0/
\ No newline at end of file
diff --git a/kubernetes/files/rc.yml b/kubernetes/files/rc.yml
index 0a31b6c..8edb271 100644
--- a/kubernetes/files/rc.yml
+++ b/kubernetes/files/rc.yml
@@ -2,10 +2,18 @@
 apiVersion: {{ service.apiVersion }}
 kind: {{ service.kind }}
 metadata:
-  name: {{ service.service }}-{{ service.role }}
-  namespace: {{ service.namespace }}
+  {%- if service.role is defined %}
   labels:
+    name: {{ service.service }}-{{ service.role }}
     app: {{ service.service }}-{{ service.role }}
+  name: {{ service.service }}-{{ service.role }}
+  {%- else %}
+  labels:
+    name: {{ service.service }}
+    app: {{ service.service }}
+  name: {{ service.service }}
+  {%- endif %}
+  namespace: {{ service.namespace }}
 spec:
   replicas: {{ service.replicas }}
   {%- if service.kind == 'PetSet' %}
@@ -14,7 +22,11 @@
   template:
     metadata:
       labels:
+        {%- if service.role is defined %}
         app: {{ service.service }}-{{ service.role }}
+        {%- else %}
+        app: {{ service.service }}
+        {%- endif %}
       annotations:
       {%- if service.hostname is defined %}
         pod.beta.kubernetes.io/hostname: {{ service.hostname }}
@@ -95,6 +107,16 @@
       {%- if service.host_pid is defined %}
       hostPID: True
       {%- endif %}
+      {%- if service.host_aliases is defined %}
+      hostAliases:
+        {%- for host_alias in service.host_aliases %}
+        - ip: {{ host_alias.ip }}
+          hostnames:
+          {%- for hostname in host_alias.hostnames %}
+            - {{ hostname }}
+          {%- endfor %}
+        {%- endfor %}
+      {%- endif %}
       containers:
       {%- for container_name, container in service.container.items() %}
         - name: {{ container_name }}
diff --git a/kubernetes/files/svc.yml b/kubernetes/files/svc.yml
index b236417..d730639 100644
--- a/kubernetes/files/svc.yml
+++ b/kubernetes/files/svc.yml
@@ -2,10 +2,17 @@
 apiVersion: v1
 kind: Service
 metadata:
+  {%- if service.role is defined %}
   labels:
     name: {{ service.service }}-{{ service.role }}
     app: {{ service.service }}-{{ service.role }}
   name: {{ service.service }}-{{ service.role }}
+  {%- else %}
+  labels:
+    name: {{ service.service }}
+    app: {{ service.service }}
+  name: {{ service.service }}
+  {%- endif %}
   namespace: {{ service.namespace }}
 spec:
   ports:
@@ -15,7 +22,11 @@
     {%- endfor %}
   type: {{ service.type }}
   selector:
+    {%- if service.role is defined %}
     app: {{ service.service }}-{{ service.role }}
+    {%- else %}
+    app: {{ service.service }}
+    {%- endif %}
   {%- if service.cluster_ip is defined %}
   clusterIP: {{ service.cluster_ip }}
   {%- endif %}
diff --git a/kubernetes/files/systemd/criproxy.service b/kubernetes/files/systemd/criproxy.service
index 9980501..2279313 100644
--- a/kubernetes/files/systemd/criproxy.service
+++ b/kubernetes/files/systemd/criproxy.service
@@ -16,7 +16,7 @@
 [Service]
 SyslogIdentifier=criproxy
 User=root
-ExecStart=/usr/bin/criproxy -alsologtostderr \
+ExecStart=/usr/bin/criproxy -logtostderr \
           {%- if common.get('containerd', {}).get('enabled') %}
           -connect /run/containerd/containerd.sock,virtlet.cloud:/run/virtlet.sock \
           {%- else %}
@@ -24,7 +24,6 @@
           {%- endif %}
           -listen /var/run/criproxy.sock \
           -v 3 \
-          -alsologtostderr \
 {%- if salt['pkg.version_cmp'](version,'1.8') < 0 %}
           -apiVersion {{ version }} \
 {%- endif %}
diff --git a/kubernetes/init.sls b/kubernetes/init.sls
index 22aad26..a882a97 100644
--- a/kubernetes/init.sls
+++ b/kubernetes/init.sls
@@ -10,4 +10,7 @@
 {%- if pillar.kubernetes.control is defined %}
 - kubernetes.control
 {%- endif %}
+{%- if pillar.kubernetes.client is defined %}
+- kubernetes.client
+{%- endif %}
 {%- endif %}
diff --git a/kubernetes/map.jinja b/kubernetes/map.jinja
index 615404c..3c3d4cb 100644
--- a/kubernetes/map.jinja
+++ b/kubernetes/map.jinja
@@ -111,6 +111,9 @@
     },
 }, merge=salt['pillar.get']('kubernetes:control')) %}
 
+{% set client = salt['grains.filter_by']({
+}, merge=salt['pillar.get']('kubernetes:client')) %}
+
 {%- set monitoring = salt['grains.filter_by']({
   'default': {
     'instance_minor_threshold_percent': 0.3,
diff --git a/kubernetes/master/controller.sls b/kubernetes/master/controller.sls
index a458868..0bfe698 100644
--- a/kubernetes/master/controller.sls
+++ b/kubernetes/master/controller.sls
@@ -231,6 +231,11 @@
         --root-ca-file=/etc/kubernetes/ssl/ca-{{ master.ca }}.crt
         --service-account-private-key-file=/etc/kubernetes/ssl/kubernetes-server.key
         --use-service-account-credentials
+{%- if common.get('cloudprovider', {}).get('enabled') and common.get('cloudprovider', {}).get('provider') == 'openstack' %}
+        --external-cloud-volume-plugin=openstack
+        --cloud-config /etc/kubernetes/cloud-config.intree
+        --cloud-provider external
+{%- endif %}
         --v={{ master.get('verbosity', 2) }}
 {%- if master.network.get('flannel', {}).get('enabled', False) %}
         --allocate-node-cidrs=true
@@ -317,11 +322,19 @@
     - file: /etc/kubernetes/cloud-config
     - file: /etc/default/openstack-cloud-controller-manager
     - file: /etc/kubernetes/controller-manager.kubeconfig
+    - file: /usr/bin/openstack-cloud-controller-manager
+
+kube_controller_mnanager_service:
+  service.running:
+  - name: kube-controller-manager
+  - watch:
+    - file: /etc/kubernetes/cloud-config.intree
 {%- endif %}
 {%- endif %}
 
 {%- endif %}
 
+{%- if master.namespace is defined %}
 
 {%- for name,namespace in master.namespace.items() %}
 
@@ -332,7 +345,11 @@
 kubernetes_namespace_create_{{ name }}:
   cmd.run:
     - name: kubectl create ns "{{ name }}"
-    - name: kubectl get ns -o=custom-columns=NAME:.metadata.name | grep -v NAME | grep "{{ name }}" > /dev/null || kubectl create ns "{{ name }}"
+    - unless: kubectl get ns -o=custom-columns=NAME:.metadata.name | grep -v NAME | grep "{{ name }}"
+    - retry:
+        attempts: 3
+        until: True
+        interval: 10
     {%- if grains.get('noservices') %}
     - onlyif: /bin/false
     {%- endif %}
@@ -341,12 +358,19 @@
 
 kubernetes_namespace_delete_{{ name }}:
   cmd.run:
-    - name: kubectl get ns -o=custom-columns=NAME:.metadata.name | grep -v NAME | grep "{{ name }}" > /dev/null && kubectl delete ns "{{ name }} || true"
+    - name: kubectl delete ns "{{ name }}"
+    - onlyif:
+      - kubectl get ns -o=custom-columns=NAME:.metadata.name | grep -v NAME | grep "{{ name }}" > /dev/null
+      {%- if grains.get('noservices') %}
+      - /bin/false
+      {%- endif %}
 
 {%- endif %}
 
 {%- endfor %}
 
+{%- endif %}
+
 {%- if master.registry.secret is defined %}
 
 {%- for name,registry in master.registry.secret.items() %}
diff --git a/kubernetes/meta/fluentd.yml b/kubernetes/meta/fluentd.yml
index 623fcea..3a88acf 100644
--- a/kubernetes/meta/fluentd.yml
+++ b/kubernetes/meta/fluentd.yml
@@ -1,22 +1,21 @@
-{%- from "kubernetes/map.jinja" import common with context -%}
+{%- from "kubernetes/map.jinja" import common, master, pool with context -%}
 {%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
-{%- from "kubernetes/map.jinja" import pool, master %}
-{%- if pool.get('enabled', False) %}
-{% set network = pool.get('network', {}) %}
-{%- else %}
-{%- if master.get('enabled', False) %}
-{% set network = master.get('network', {}) %}
-{% endif %}
-{% endif %}
+  {%- if pool.get('enabled', False) %}
+    {% set network = pool.get('network', {}) %}
+  {%- else %}
+    {%- if master.get('enabled', False) %}
+      {% set network = master.get('network', {}) %}
+    {% endif %}
+  {% endif %}
 
-{%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
+  {%- set positiondb = pillar.fluentd.agent.dir.positiondb %}
 agent:
   plugin:
     fluent-plugin-kubernetes_metadata_filter:
       deb: ['td-agent-additional-plugins']
   config:
     label:
-      {%- if pillar.docker is defined %}
+  {%- if pillar.docker is defined %}
       docker:
         filter:
           add_drop_tag:
@@ -26,7 +25,7 @@
             record:
               - name: drop_event
                 value: ${ record.fetch('attrs', {}).fetch('io.kubernetes.pod.name', '') }
-      {%- endif %}
+  {%- endif %}
       kubernetes:
         input:
           container:
@@ -36,10 +35,16 @@
             path_key: log_path
             pos_file: {{ positiondb }}/kubernetes.pos
             parser:
+  {%- if pillar.docker is defined %}
               type: json
+  {%- else %}
+              # Containerd CRI log format https://regex101.com/r/BAw3bQ/1
+              type: regexp
+              format: /^(?<time>.+) (?<stream>stdout|stderr)( (?<logtag>.))? (?<Payload>.*)$/
+  {%- endif %}
               time_format: '%Y-%m-%dT%H:%M:%S.%NZ'
               keep_time_key: false
-          {%- if network is defined and network.get('calico', {}).get('enabled', False) %}
+  {%- if network is defined and network.get('calico', {}).get('enabled', False) %}
           bird:
             type: tail
             tag: kubernetes.calico.bird
@@ -76,19 +81,19 @@
               time_key: Timestamp
               keep_time_key: false
               format: '/^(?<Timestamp>[^ ]+ [^ ]+) \[(?<orig_severity_label>[^ ]+)\]\[\d+?\] (?<Payload>.*)$/'
-          {%- endif %}
+  {%- endif %}
         filter:
           add_kubernetes_meta:
             tag: 'temp.kubernetes.container.**'
             type: kubernetes_metadata
             kubernetes_url: https://{{ pool.apiserver.host }}:{{ pool.apiserver.secure_port }}
-            {%- if common.get('cloudprovider', {}).get('enabled') and common.get('cloudprovider', {}).get('provider') == 'openstack' %}
+  {%- if common.get('cloudprovider', {}).get('enabled') and common.get('cloudprovider', {}).get('provider') == 'openstack' %}
             client_cert: /etc/kubernetes/ssl/kubelet-client-fqdn.crt
             client_key: /etc/kubernetes/ssl/kubelet-client-fqdn.key
-            {%- else %}
+  {%- else %}
             client_cert: /etc/kubernetes/ssl/kubelet-client.crt
             client_key: /etc/kubernetes/ssl/kubelet-client.key
-            {%- endif %}
+  {%- endif %}
             ca_file: /etc/kubernetes/ssl/ca-kubernetes.crt
             verify_ssl: True
           enrich_container:
@@ -104,7 +109,7 @@
                 value: 6
               - name: programname
                 value: ${ record['kubernetes']['container_name'] }
-          {%- if network is defined and network.get('calico', {}).get('enabled', False) %}
+  {%- if network is defined and network.get('calico', {}).get('enabled', False) %}
           enrich_bird:
             tag: 'kubernetes.calico.bird'
             type: record_transformer
@@ -140,7 +145,7 @@
                 value: ${ {'DEBUG'=>7,'INFO'=>6,'WARNING'=>4,'ERROR'=>3,'FATAL'=>2,'PANIC'=>1}[record['orig_severity_label']].to_i }
               - name: programname
                 value: calico-felix
-          {%- endif %}
+  {%- endif %}
         match:
           cast_service_tag:
             tag: 'temp.kubernetes.container.**'
diff --git a/kubernetes/pool/images.sls b/kubernetes/pool/images.sls
new file mode 100644
index 0000000..f58f547
--- /dev/null
+++ b/kubernetes/pool/images.sls
@@ -0,0 +1,14 @@
+{%- from "kubernetes/map.jinja" import pool with context %}
+{%- if pool.get('enabled', False) and pool.get('images', {}) %}
+
+{%- for image in pool.get('images', []) %}
+
+{{ image }}_image:
+  cmd.run:
+    - name: /usr/local/bin/crictl pull {{ image }}
+    - onlyif: "test -e /usr/local/bin/crictl"
+    - unless: "/usr/local/bin/crictl images -o yaml | grep {{ image }}"
+
+{%- endfor %}
+
+{%- endif %}
\ No newline at end of file
diff --git a/kubernetes/pool/init.sls b/kubernetes/pool/init.sls
index d5acd23..8a0919f 100644
--- a/kubernetes/pool/init.sls
+++ b/kubernetes/pool/init.sls
@@ -23,3 +23,6 @@
 {%- if common.addons.get('virtlet', {}).get('use_apparmor') and not pillar.get('kubernetes', {}).get('master', False) %}
 - kubernetes.pool.virtlet-apparmor
 {%- endif %}
+{%- if pool.get('images', {}) %}
+- kubernetes.pool.images
+{%- endif %}
diff --git a/metadata/service/master/cluster.yml b/metadata/service/master/cluster.yml
index afeb317..c611cda 100644
--- a/metadata/service/master/cluster.yml
+++ b/metadata/service/master/cluster.yml
@@ -47,9 +47,6 @@
       ca: kubernetes
       storage:
         engine: none
-      namespace:
-        kube-system:
-          enabled: True
       network:
         calico:
           etcd:
diff --git a/metadata/service/master/single.yml b/metadata/service/master/single.yml
index be5d701..a9314b1 100644
--- a/metadata/service/master/single.yml
+++ b/metadata/service/master/single.yml
@@ -65,9 +65,6 @@
       ca: kubernetes
       storage:
         engine: none
-      namespace:
-        kube-system:
-          enabled: True
       network:
         calico:
           etcd: