Linux OVS-dpdk and multiqueue support
Introduce dpdk support for linux OVS configuration.
It configures dpdk interface bind, ovs dpdk ports, bonding,
parameters for dpdk cpu pmd and set multique queues for specific
ovs dpdk interfaces.
Change-Id: I3f38660bab8db0c2b38f03ed8c94eb10b6b3beb9
Epic: PROD-8957
Epic: PROD-8958
diff --git a/README.rst b/README.rst
index 7f38c2a..39893c1 100644
--- a/README.rst
+++ b/README.rst
@@ -652,19 +652,92 @@
network:
resolv:
dns:
- - 8.8.4.4
- - 8.8.8.8
+ - 8.8.4.4
+ - 8.8.8.8
domain: my.example.com
search:
- - my.example.com
- - example.com
+ - my.example.com
+ - example.com
options:
- - ndots:5
- - timeout:2
- - attempts:2
+ - ndots: 5
+ - timeout: 2
+ - attempts: 2
-Linux storage pillars
----------------------
+DPDK OVS interfaces
+--------------------
+
+**DPDK OVS NIC**
+
+.. code-block:: yaml
+
+ linux:
+ network:
+ bridge: openvswitch
+ dpdk:
+ enabled: true
+ driver: uio/vfio-pci
+ openvswitch:
+ pmd_cpu_mask: "0x6"
+ dpdk_socket_mem: "1024,1024"
+ dpdk_lcore_mask: "0x400"
+ memory_channels: 2
+ interface:
+ dpkd0:
+ name: ${_param:dpdk_nic}
+ pci: 0000:06:00.0
+ driver: igb_uio/vfio
+ enabled: true
+ type: dpdk_ovs_port
+ n_rxq: 2
+ bridge: br-prv
+ br-prv:
+ enabled: true
+ type: dpdk_ovs_bridge
+
+**DPDK OVS Bond**
+
+.. code-block:: yaml
+
+ linux:
+ network:
+ bridge: openvswitch
+ dpdk:
+ enabled: true
+ driver: uio/vfio-pci
+ openvswitch:
+ pmd_cpu_mask: "0x6"
+ dpdk_socket_mem: "1024,1024"
+ dpdk_lcore_mask: "0x400"
+ memory_channels: 2
+ interface:
+ dpdk_second_nic:
+ name: ${_param:primary_second_nic}
+ pci: 0000:06:00.0
+ driver: igb_uio/vfio
+ bond: dpdkbond0
+ enabled: true
+ type: dpdk_ovs_port
+ n_rxq: 2
+ dpdk_first_nic:
+ name: ${_param:primary_first_nic}
+ pci: 0000:05:00.0
+ driver: igb_uio/vfio
+ bond: dpdkbond0
+ enabled: true
+ type: dpdk_ovs_port
+ n_rxq: 2
+ dpdkbond0:
+ enabled: true
+ bridge: br-prv
+ type: dpdk_ovs_bond
+ mode: active-backup
+ br-prv:
+ enabled: true
+ type: dpdk_ovs_bridge
+
+
+Linux storage
+-------------
Linux with mounted Samba
@@ -681,7 +754,7 @@
- file_system: cifs
- options: guest,uid=myuser,iocharset=utf8,file_mode=0777,dir_mode=0777,noperm
-Linux with file swap
+File swap configuration
.. code-block:: yaml
@@ -695,7 +768,7 @@
device: /swapfile
size: 1024
-Linux with partition swap
+Partition swap configuration
.. code-block:: yaml
diff --git a/linux/files/dpdk_interfaces b/linux/files/dpdk_interfaces
new file mode 100644
index 0000000..97ebf30
--- /dev/null
+++ b/linux/files/dpdk_interfaces
@@ -0,0 +1,6 @@
+{%- from "linux/map.jinja" import network with context %}
+{%- for interface_name, interface in network.interface.iteritems() %}
+{%- if 'dpdk' in interface.type and interface.pci is defined %}
+pci {{ interface.pci }} {{ interface.driver }}
+{%- endif %}
+{%- endfor %}
diff --git a/linux/map.jinja b/linux/map.jinja
index 8329be8..b49ff66 100644
--- a/linux/map.jinja
+++ b/linux/map.jinja
@@ -11,7 +11,7 @@
'repo': {},
'package': {},
'autoupdates': {
- 'pkgs': []
+ 'pkgs': []
},
'selinux': 'permissive',
'ca_certs_dir': '/usr/local/share/ca-certificates',
@@ -106,6 +106,7 @@
'hostname_file': '/etc/hostname',
'bridge_pkgs': ['bridge-utils'],
'ovs_pkgs': ['openvswitch-switch', 'bridge-utils'],
+ 'dpdk_pkgs': ['dpdk', 'dpdk-dev', 'dpdk-dkms', 'dpdk-igb-uio-dkms', 'dpdk-rte-kni-dkms'],
'network_manager': False,
'interface': {},
'interface_params': interface_params,
diff --git a/linux/network/dpdk.sls b/linux/network/dpdk.sls
new file mode 100644
index 0000000..78d7ab2
--- /dev/null
+++ b/linux/network/dpdk.sls
@@ -0,0 +1,134 @@
+{%- from "linux/map.jinja" import network with context %}
+{%- from "linux/map.jinja" import system with context %}
+{%- if network.dpdk.enabled %}
+
+linux_dpdk_pkgs:
+ pkg.installed:
+ - pkgs: {{ network.dpdk_pkgs }}
+
+linux_dpdk_kernel_module:
+ kmod.present:
+ - name: {{ network.dpdk.driver }}
+ - require:
+ - pkg: linux_dpdk_pkgs
+ - require_in:
+ - service: linux_network_dpdk_service
+
+/etc/dpdk/interfaces:
+ file.managed:
+ - source: salt://linux/files/dpdk_interfaces
+ - template: jinja
+ - user: root
+ - group: root
+ - mode: 644
+ - require:
+ - pkg: linux_dpdk_pkgs
+
+linux_network_dpdk_service:
+ service.running:
+ - enable: true
+ - name: dpdk
+ - watch:
+ - file: /etc/dpdk/interfaces
+
+{%- if network.openvswitch is defined %}
+
+openvswitch_dpdk_pkgs:
+ pkg.installed:
+ - pkgs:
+ - openvswitch-switch-dpdk
+ - openvswitch-switch
+ - bridge-utils
+
+linux_network_dpdk_ovs_service:
+ cmd.run:
+ - name: "ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true"
+ - require:
+ - service: linux_network_dpdk_service
+ - unless: 'ovs-vsctl get Open_vSwitch . other_config | grep "dpdk-init=\"true\""'
+
+{%- set ovs_options = [
+ "pmd-cpu-mask=\""+network.openvswitch.pmd_cpu_mask+"\"",
+ "dpdk-socket-mem=\""+network.openvswitch.dpdk_socket_mem+"\"",
+ "dpdk-lcore-mask=\""+network.openvswitch.dpdk_lcore_mask+"\"",
+ "dpdk-extra=\"-n "+network.openvswitch.memory_channels+" --vhost-owner libvirt-qemu:kvm --vhost-perm 0664\""
+]
+%}
+
+{%- for option in ovs_options %}
+
+linux_network_dpdk_ovs_option_{{ option }}:
+ cmd.run:
+ - name: 'ovs-vsctl set Open_vSwitch . other_config:{{ option }}'
+ - watch_in:
+ - service: service_openvswitch
+ - require:
+ - cmd: linux_network_dpdk_ovs_service
+ - unless: |
+ ovs-vsctl get Open_vSwitch . other_config | grep '{{ option }}'
+
+{%- endfor %}
+
+service_openvswitch:
+ service.running:
+ - name: openvswitch-switch
+ - enable: true
+ - watch:
+ - cmd: linux_network_dpdk_ovs_service
+
+{%- endif %}
+
+{%- for interface_name, interface in network.interface.iteritems() if interface.get('enabled', True) %}
+
+ {%- if interface.type == "dpdk_ovs_bond" %}
+
+ {%- set bond_interfaces = {} %}
+ {%- for iface_name, iface in network.interface.iteritems() if iface.get('enabled', True) and iface.get('bond',"") == interface_name %}
+ {#- Get list of child interfaces #}
+ {%- do bond_interfaces.update({iface_name: iface}) %}
+ {%- endfor %}
+
+linux_network_dpdk_bond_interface_{{ interface_name }}:
+ cmd.run:
+ - name: "ovs-vsctl add-bond {{ interface.bridge }} {{ interface_name }} {{ bond_interfaces.keys()|join(' ') }} {% for iface_name, iface in bond_interfaces.iteritems() %}-- set Interface {{ iface_name }} type=dpdk options:dpdk-devargs={{ iface.pci }} {% endfor %}"
+ - unless: "ovs-vsctl show | grep {{ interface_name }}"
+ - require:
+ - cmd: linux_network_dpdk_bridge_interface_{{ interface.bridge }}
+
+linux_network_dpdk_bond_mode_{{ interface_name }}:
+ cmd.run:
+ - name: "ovs-vsctl set port {{ interface_name }} bond_mode={{ interface.mode }}"
+ - unless: "ovs-appctl bond/show {{ interface_name }} | grep {{ interface.mode }}"
+ - require:
+ - cmd: linux_network_dpdk_bond_interface_{{ interface_name }}
+
+ {%- elif interface.type == 'dpdk_ovs_bridge' %}
+
+linux_network_dpdk_bridge_interface_{{ interface_name }}:
+ cmd.run:
+ - name: "ovs-vsctl add-br {{ interface_name }} -- set bridge {{ interface_name }} datapath_type=netdev"
+ - unless: "ovs-vsctl show | grep {{ interface_name }}"
+
+ {%- elif interface.type == 'dpdk_ovs_port' and interface.bridge is defined %}
+
+linux_network_dpdk_bridge_port_interface_{{ interface_name }}:
+ cmd.run:
+ - name: "ovs-vsctl add-port {{ interface.bridge }} dpdk0 -- set Interface dpdk0 type=dpdk options:dpdk-devargs={{ interface.pci }}"
+ - unless: "ovs-vsctl show | grep dpdk0"
+ - require:
+ - cmd: linux_network_dpdk_bridge_interface_{{ interface.bridge }}
+
+ {# Multiqueue n_rxq setup on interfaces #}
+ {%- elif interface.type == 'dpdk_ovs_port' and interface.n_rxq is defined %}
+
+linux_network_dpdk_bridge_port_interface_n_rxq_{{ interface_name }}:
+ cmd.run:
+ - name: "ovs-vsctl set Interface {{ interface_name }} options:n_rxq={{ interface.n_rxq }} "
+ - unless: |
+ ovs-vsctl get Interface {{ interface_name }} options | grep 'n_rxq="{{ interface.n_rxq }}"'
+
+ {%- endif %}
+
+{%- endfor %}
+
+{%- endif %}
diff --git a/linux/network/init.sls b/linux/network/init.sls
index cdef5a7..53d394b 100644
--- a/linux/network/init.sls
+++ b/linux/network/init.sls
@@ -7,6 +7,9 @@
{%- if network.resolv is defined %}
- linux.network.resolv
{%- endif %}
+{%- if network.dpdk is defined %}
+- linux.network.dpdk
+{%- endif %}
{%- if network.interface|length > 0 %}
- linux.network.interface
{%- endif %}
diff --git a/linux/network/interface.sls b/linux/network/interface.sls
index c0f8154..0b1f0ee 100644
--- a/linux/network/interface.sls
+++ b/linux/network/interface.sls
@@ -24,7 +24,8 @@
{%- set interface_name = interface.get('name', interface_name) %}
-{%- if interface.get('managed', True) %}
+{# it is not used for any interface with type preffix dpdk,eg. dpdk_ovs_port #}
+{%- if interface.get('managed', True) and not 'dpdk' in interface.type %}
{%- if grains.os_family in ['RedHat', 'Debian'] %}
diff --git a/linux/network/resolv.sls b/linux/network/resolv.sls
index 89fb6f6..db1a463 100644
--- a/linux/network/resolv.sls
+++ b/linux/network/resolv.sls
@@ -9,7 +9,7 @@
linux_resolvconf_disable:
cmd.run:
- - name: resolvconf --disable-updates
- - onlyif: resolvconf --updates-are-enabled
+ - name: resolvconf --disable-updates
+ - onlyif: resolvconf --updates-are-enabled
{%- endif %}
diff --git a/linux/system/hugepages.sls b/linux/system/hugepages.sls
index e8db507..e382e51 100644
--- a/linux/system/hugepages.sls
+++ b/linux/system/hugepages.sls
@@ -24,7 +24,7 @@
- device: Hugetlbfs-kvm
- fstype: hugetlbfs
- mkmnt: true
- - opts: mode=777,pagesize={{ hugepages.size }}
+ - opts: mode=775,gid=kvm,pagesize={{ hugepages.size }}
{%- endif %}
diff --git a/tests/pillar/network_openvswitch_dpdk.sls b/tests/pillar/network_openvswitch_dpdk.sls
new file mode 100644
index 0000000..73623d6
--- /dev/null
+++ b/tests/pillar/network_openvswitch_dpdk.sls
@@ -0,0 +1,41 @@
+linux:
+ system:
+ enabled: true
+ domain: local
+ network:
+ enabled: true
+ hostname: test01
+ fqdn: test01.local
+ network_manager: false
+ bridge: openvswitch
+ dpdk:
+ enabled: true
+ driver: uio
+ openvswitch:
+ pmd_cpu_mask: "0x6"
+ dpdk_socket_mem: "1024"
+ dpdk_lcore_mask: "0x400"
+ memory_channels: "2"
+ interface:
+ dpdk0:
+ name: enp5s0f1
+ pci: "0000:05:00.1"
+ driver: igb_uio
+ bond: dpdkbond0
+ enabled: true
+ type: dpdk_ovs_port
+ dpdk1:
+ name: enp5s0f2
+ pci: "0000:05:00.2"
+ driver: igb_uio
+ bond: dpdkbond0
+ enabled: true
+ type: dpdk_ovs_port
+ dpdkbond0:
+ enabled: true
+ bridge: br-prv
+ type: dpdk_ovs_bond
+ mode: active-backup
+ br-prv:
+ enabled: true
+ type: dpdk_ovs_bridge