From: http://xenbits.xen.org/XCP/linux-2.6.32.pq.hg?rev/20e4634f7b7b
Subject: apply xen specific patch to the Chelsio ethernet drivers
as a result of their feedback from the Cowly Beta
Patch-mainline: n/a
* Disable LRO by default. The kernel.org driver does enable it, but it
does not play very well with the bridging layer. (Please note that the
kernel.org driver does now implement GRO)
* Allocate SKBs instead of pages for incoming data. Using pages causes
traffic to stall when the VMs use large MTUs.
* Disable lazy completion to Tx buffers. cxgb3 completion mechanism
coalesces TX completion notifications, but this breaks the VM's
behavior: The VMs networking stacks rely on skb to be freed in the
hypervisor to open the Tx buffer.
Acked-by: bphilips@suse.de
--- head.orig/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c 2012-05-08 10:45:51.000000000 +0200
+++ head/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c 2012-05-08 11:19:52.000000000 +0200
@@ -3313,7 +3313,17 @@ static int __devinit init_one(struct pci
* register at least one net device.
*/
for_each_port(adapter, i) {
+#ifndef CONFIG_XEN
err = register_netdev(adapter->port[i]);
+#else
+ rtnl_lock();
+ err = register_netdevice(adapter->port[i]);
+ if (!err) {
+ adapter->port[i]->wanted_features &= ~NETIF_F_GRO;
+ netdev_update_features(adapter->port[i]);
+ }
+ rtnl_unlock();
+#endif
if (err)
dev_warn(&pdev->dev,
"cannot register net device %s, skipping\n",
--- head.orig/drivers/net/ethernet/chelsio/cxgb3/sge.c 2012-05-08 10:45:51.000000000 +0200
+++ head/drivers/net/ethernet/chelsio/cxgb3/sge.c 2011-09-09 15:54:39.000000000 +0200
@@ -59,11 +59,24 @@
* It must be a divisor of PAGE_SIZE. If set to 0 FL0 will use sk_buffs
* directly.
*/
+#ifndef CONFIG_XEN
#define FL0_PG_CHUNK_SIZE 2048
+#else
+/* Use skbuffs for XEN kernels. LRO is already disabled */
+#define FL0_PG_CHUNK_SIZE 0
+#endif
+
#define FL0_PG_ORDER 0
#define FL0_PG_ALLOC_SIZE (PAGE_SIZE << FL0_PG_ORDER)
+
+#ifndef CONFIG_XEN
#define FL1_PG_CHUNK_SIZE (PAGE_SIZE > 8192 ? 16384 : 8192)
#define FL1_PG_ORDER (PAGE_SIZE > 8192 ? 0 : 1)
+#else
+#define FL1_PG_CHUNK_SIZE 0
+#define FL1_PG_ORDER 0
+#endif
+
#define FL1_PG_ALLOC_SIZE (PAGE_SIZE << FL1_PG_ORDER)
#define SGE_RX_DROP_THRES 16
@@ -1268,7 +1281,27 @@ netdev_tx_t t3_eth_xmit(struct sk_buff *
gen = q->gen;
q->unacked += ndesc;
+#ifdef CONFIG_XEN
+ /*
+ * Some Guest OS clients get terrible performance when they have bad
+ * message size / socket send buffer space parameters. For instance,
+ * if an application selects an 8KB message size and an 8KB send
+ * socket buffer size. This forces the application into a single
+ * packet stop-and-go mode where it's only willing to have a single
+ * message outstanding. The next message is only sent when the
+ * previous message is noted as having been sent. Until we issue a
+ * kfree_skb() against the TX skb, the skb is charged against the
+ * application's send buffer space. We only free up TX skbs when we
+ * get a TX credit return from the hardware / firmware which is fairly
+ * lazy about this. So we request a TX WR Completion Notification on
+ * every TX descriptor in order to accellerate TX credit returns. See
+ * also the change in handle_rsp_cntrl_info() to free up TX skb's when
+ * we receive the TX WR Completion Notifications ...
+ */
+ compl = F_WR_COMPL;
+#else
compl = (q->unacked & 8) << (S_WR_COMPL - 3);
+#endif
q->unacked &= 7;
pidx = q->pidx;
q->pidx += ndesc;
@@ -2154,8 +2187,35 @@ static inline void handle_rsp_cntrl_info
#endif
credits = G_RSPD_TXQ0_CR(flags);
- if (credits)
+ if (credits) {
qs->txq[TXQ_ETH].processed += credits;
+#ifdef CONFIG_XEN
+ /*
+ * In the normal Linux driver t3_eth_xmit() routine, we call
+ * skb_orphan() on unshared TX skb. This results in a call to
+ * the destructor for the skb which frees up the send buffer
+ * space it was holding down. This, in turn, allows the
+ * application to make forward progress generating more data
+ * which is important at 10Gb/s. For Virtual Machine Guest
+ * Operating Systems this doesn't work since the send buffer
+ * space is being held down in the Virtual Machine. Thus we
+ * need to get the TX skb's freed up as soon as possible in
+ * order to prevent applications from stalling.
+ *
+ * This code is largely copied from the corresponding code in
+ * sge_timer_tx() and should probably be kept in sync with any
+ * changes there.
+ */
+ if (__netif_tx_trylock(qs->tx_q)) {
+ struct port_info *pi = netdev_priv(qs->netdev);
+ struct adapter *adap = pi->adapter;
+
+ reclaim_completed_tx(adap, &qs->txq[TXQ_ETH],
+ TX_RECLAIM_CHUNK);
+ __netif_tx_unlock(qs->tx_q);
+ }
+#endif
+ }
credits = G_RSPD_TXQ2_CR(flags);
if (credits)
--- head.orig/drivers/net/ethernet/chelsio/cxgb3/version.h 2012-05-08 10:45:51.000000000 +0200
+++ head/drivers/net/ethernet/chelsio/cxgb3/version.h 2012-04-11 17:11:24.000000000 +0200
@@ -35,7 +35,11 @@
#define DRV_DESC "Chelsio T3 Network Driver"
#define DRV_NAME "cxgb3"
/* Driver version */
+#ifndef CONFIG_XEN
#define DRV_VERSION "1.1.5-ko"
+#else
+#define DRV_VERSION "1.1.5-xen-ko"
+#endif
/* Firmware version */
#define FW_VERSION_MAJOR 7