netdev-dpdk: Allow configurable queue sizes for 'dpdk' ports

The 'options:n_rxq_desc' and 'n_txq_desc' fields allow the number of rx and tx descriptors for dpdk ports to be modified. By default the values are set to 2048, but can be modified to an integer between 1 and 4096 that is a power of two. The values can be modified at runtime, however require the NIC to restart when changed. Signed-off-by: Ciara Loftus <[email protected]> Acked-by: Yunhong Jiang <[email protected]> Acked-by: Ilya Maximets <[email protected]> Signed-off-by: Daniele Di Proietto <[email protected]>
water0615 · Sep 30, 2016 · b685696 · b685696
1 parent ff8802a
commit b685696
Show file tree

Hide file tree

Showing 4 changed files with 96 additions and 9 deletions.
diff --git a/INSTALL.DPDK-ADVANCED.md b/INSTALL.DPDK-ADVANCED.md
@@ -257,7 +257,23 @@ needs to be affinitized accordingly.
   The rx queues are assigned to pmd threads on the same NUMA node in a
   round-robin fashion.
 
-### 4.4 Exact Match Cache
+### 4.4 DPDK Physical Port Queue Sizes
+  `ovs-vsctl set Interface dpdk0 options:n_rxq_desc=<integer>`
+  `ovs-vsctl set Interface dpdk0 options:n_txq_desc=<integer>`
+
+  The command above sets the number of rx/tx descriptors that the NIC
+  associated with dpdk0 will be initialised with.
+
+  Different 'n_rxq_desc' and 'n_txq_desc' configurations yield different
+  benefits in terms of throughput and latency for different scenarios.
+  Generally, smaller queue sizes can have a positive impact for latency at the
+  expense of throughput. The opposite is often true for larger queue sizes.
+  Note: increasing the number of rx descriptors eg. to 4096  may have a
+  negative impact on performance due to the fact that non-vectorised DPDK rx
+  functions may be used. This is dependant on the driver in use, but is true
+  for the commonly used i40e and ixgbe DPDK drivers.
+
+### 4.5 Exact Match Cache
 
   Each pmd thread contains one EMC. After initial flow setup in the
   datapath, the EMC contains a single table and provides the lowest level
@@ -274,7 +290,7 @@ needs to be affinitized accordingly.
   avoiding datapath classifier lookups is to have multiple pmd threads
   running. This can be done as described in section 4.2.
 
-### 4.5 Rx Mergeable buffers
+### 4.6 Rx Mergeable buffers
 
   Rx Mergeable buffers is a virtio feature that allows chaining of multiple
   virtio descriptors to handle large packet sizes. As such, large packets

diff --git a/NEWS b/NEWS
@@ -127,6 +127,8 @@ v2.6.0 - xx xxx xxxx
      * Remove dpdkvhostcuse port type.
      * OVS client mode for vHost and vHost reconnect (Requires QEMU 2.7)
      * 'dpdkvhostuserclient' port type.
+     * New option 'n_rxq_desc' and 'n_txq_desc' fields for DPDK interfaces
+       which set the number of rx and tx descriptors to use for the given port.
    - Increase number of registers to 16.
    - ovs-benchmark: This utility has been removed due to lack of use and
      bitrot.

diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
@@ -134,10 +134,12 @@ BUILD_ASSERT_DECL((MAX_NB_MBUF / ROUND_DOWN_POW2(MAX_NB_MBUF/MIN_NB_MBUF))
 
 #define SOCKET0              0
 
-/* Size of Physical NIC RX Queue, Max (n + 32 <= 4096) */
-#define NIC_PORT_RX_Q_SIZE 2048
-/* Size of Physical NIC TX Queue, Max (n + 32 <= 4096) */
-#define NIC_PORT_TX_Q_SIZE 2048
+/* Default size of Physical NIC RXQ */
+#define NIC_PORT_DEFAULT_RXQ_SIZE 2048
+/* Default size of Physical NIC TXQ */
+#define NIC_PORT_DEFAULT_TXQ_SIZE 2048
+/* Maximum size of Physical NIC Queues */
+#define NIC_PORT_MAX_Q_SIZE 4096
 
 #define OVS_VHOST_MAX_QUEUE_NUM 1024  /* Maximum number of vHost TX queues. */
 #define OVS_VHOST_QUEUE_MAP_UNKNOWN (-1) /* Mapping not initialized. */
@@ -379,6 +381,12 @@ struct netdev_dpdk {
     int requested_mtu;
     int requested_n_txq;
     int requested_n_rxq;
+    int requested_rxq_size;
+    int requested_txq_size;
+
+    /* Number of rx/tx descriptors for physical devices */
+    int rxq_size;
+    int txq_size;
 
     /* Socket ID detected when vHost device is brought up */
     int requested_socket_id;
@@ -662,7 +670,7 @@ dpdk_eth_dev_queue_setup(struct netdev_dpdk *dev, int n_rxq, int n_txq)
         }
 
         for (i = 0; i < n_txq; i++) {
-            diag = rte_eth_tx_queue_setup(dev->port_id, i, NIC_PORT_TX_Q_SIZE,
+            diag = rte_eth_tx_queue_setup(dev->port_id, i, dev->txq_size,
                                           dev->socket_id, NULL);
             if (diag) {
                 VLOG_INFO("Interface %s txq(%d) setup error: %s",
@@ -678,7 +686,7 @@ dpdk_eth_dev_queue_setup(struct netdev_dpdk *dev, int n_rxq, int n_txq)
         }
 
         for (i = 0; i < n_rxq; i++) {
-            diag = rte_eth_rx_queue_setup(dev->port_id, i, NIC_PORT_RX_Q_SIZE,
+            diag = rte_eth_rx_queue_setup(dev->port_id, i, dev->rxq_size,
                                           dev->socket_id, NULL,
                                           dev->dpdk_mp->mp);
             if (diag) {
@@ -854,6 +862,10 @@ netdev_dpdk_init(struct netdev *netdev, unsigned int port_no,
     netdev->n_txq = NR_QUEUE;
     dev->requested_n_rxq = netdev->n_rxq;
     dev->requested_n_txq = netdev->n_txq;
+    dev->rxq_size = NIC_PORT_DEFAULT_RXQ_SIZE;
+    dev->txq_size = NIC_PORT_DEFAULT_TXQ_SIZE;
+    dev->requested_rxq_size = dev->rxq_size;
+    dev->requested_txq_size = dev->txq_size;
 
     /* Initialize the flow control to NULL */
     memset(&dev->fc_conf, 0, sizeof dev->fc_conf);
@@ -1069,6 +1081,12 @@ netdev_dpdk_get_config(const struct netdev *netdev, struct smap *args)
     smap_add_format(args, "configured_rx_queues", "%d", netdev->n_rxq);
     smap_add_format(args, "requested_tx_queues", "%d", dev->requested_n_txq);
     smap_add_format(args, "configured_tx_queues", "%d", netdev->n_txq);
+    smap_add_format(args, "requested_rxq_descriptors", "%d",
+                    dev->requested_rxq_size);
+    smap_add_format(args, "configured_rxq_descriptors", "%d", dev->rxq_size);
+    smap_add_format(args, "requested_txq_descriptors", "%d",
+                    dev->requested_txq_size);
+    smap_add_format(args, "configured_txq_descriptors", "%d", dev->txq_size);
     smap_add_format(args, "mtu", "%d", dev->mtu);
     ovs_mutex_unlock(&dev->mutex);
 
@@ -1087,6 +1105,23 @@ dpdk_set_rxq_config(struct netdev_dpdk *dev, const struct smap *args)
     }
 }
 
+static void
+dpdk_process_queue_size(struct netdev *netdev, const struct smap *args,
+                        const char *flag, int default_size, int *new_size)
+{
+    int queue_size = smap_get_int(args, flag, default_size);
+
+    if (queue_size <= 0 || queue_size > NIC_PORT_MAX_Q_SIZE
+            || !is_pow2(queue_size)) {
+        queue_size = default_size;
+    }
+
+    if (queue_size != *new_size) {
+        *new_size = queue_size;
+        netdev_request_reconfigure(netdev);
+    }
+}
+
 static int
 netdev_dpdk_set_config(struct netdev *netdev, const struct smap *args)
 {
@@ -1096,6 +1131,13 @@ netdev_dpdk_set_config(struct netdev *netdev, const struct smap *args)
 
     dpdk_set_rxq_config(dev, args);
 
+    dpdk_process_queue_size(netdev, args, "n_rxq_desc",
+                            NIC_PORT_DEFAULT_RXQ_SIZE,
+                            &dev->requested_rxq_size);
+    dpdk_process_queue_size(netdev, args, "n_txq_desc",
+                            NIC_PORT_DEFAULT_TXQ_SIZE,
+                            &dev->requested_txq_size);
+
     /* Flow control support is only available for DPDK Ethernet ports. */
     bool rx_fc_en = false;
     bool tx_fc_en = false;
@@ -2945,7 +2987,9 @@ netdev_dpdk_reconfigure(struct netdev *netdev)
 
     if (netdev->n_txq == dev->requested_n_txq
         && netdev->n_rxq == dev->requested_n_rxq
-        && dev->mtu == dev->requested_mtu) {
+        && dev->mtu == dev->requested_mtu
+        && dev->rxq_size == dev->requested_rxq_size
+        && dev->txq_size == dev->requested_txq_size) {
         /* Reconfiguration is unnecessary */
 
         goto out;
@@ -2960,6 +3004,9 @@ netdev_dpdk_reconfigure(struct netdev *netdev)
     netdev->n_txq = dev->requested_n_txq;
     netdev->n_rxq = dev->requested_n_rxq;
 
+    dev->rxq_size = dev->requested_rxq_size;
+    dev->txq_size = dev->requested_txq_size;
+
     rte_free(dev->tx_q);
     err = dpdk_eth_dev_init(dev);
     netdev_dpdk_alloc_txq(dev, netdev->n_txq);

diff --git a/vswitchd/vswitch.xml b/vswitchd/vswitch.xml
@@ -2332,6 +2332,28 @@
           Only supported by dpdkvhostuserclient interfaces.
         </p>
       </column>
+
+      <column name="options" key="n_rxq_desc"
+              type='{"type": "integer", "minInteger": 1, "maxInteger": 4096}'>
+        <p>
+          Specifies the rx queue size (number rx descriptors) for dpdk ports.
+          The value must be a power of 2, less than 4096 and supported
+          by the hardware of the device being configured.
+          If not specified or an incorrect value is specified, 2048 rx
+          descriptors will be used by default.
+        </p>
+      </column>
+
+      <column name="options" key="n_txq_desc"
+              type='{"type": "integer", "minInteger": 1, "maxInteger": 4096}'>
+        <p>
+          Specifies the tx queue size (number tx descriptors) for dpdk ports.
+          The value must be a power of 2, less than 4096 and supported
+          by the hardware of the device being configured.
+          If not specified or an incorrect value is specified, 2048 tx
+          descriptors will be used by default.
+        </p>
+      </column>
     </group>
 
     <group title="MTU">