ixgbe: convert Rx to libeth and page_pool

drivers/net/ethernet/intel/Kconfig

@@ -146,6 +146,7 @@ config IXGBE
 	tristate "Intel(R) 10GbE PCI Express adapters support"
 	depends on PCI
 	depends on PTP_1588_CLOCK_OPTIONAL
+	select LIBETH_XDP
 	select LIBIE_FWLOG if DEBUG_FS
 	select MDIO
 	select NET_DEVLINK

drivers/net/ethernet/intel/ixgbe/ixgbe.h

@@ -33,6 +33,8 @@
 #include "ixgbe_ipsec.h"
 
 #include <net/xdp.h>
+#include <net/libeth/rx.h>
+#include <net/libeth/types.h>
 
 /* common prefix used by pr_<> macros */
 #undef pr_fmt
@@ -69,65 +71,14 @@
 
 /* Supported Rx Buffer Sizes */
 #define IXGBE_RXBUFFER_256    256  /* Used for skb receive header */
-#define IXGBE_RXBUFFER_1536  1536
-#define IXGBE_RXBUFFER_2K    2048
 #define IXGBE_RXBUFFER_3K    3072
 #define IXGBE_RXBUFFER_4K    4096
 #define IXGBE_MAX_RXBUFFER  16384  /* largest size for a single descriptor */
 
-#define IXGBE_PKT_HDR_PAD   (ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2))
-
-/* Attempt to maximize the headroom available for incoming frames.  We
- * use a 2K buffer for receives and need 1536/1534 to store the data for
- * the frame.  This leaves us with 512 bytes of room.  From that we need
- * to deduct the space needed for the shared info and the padding needed
- * to IP align the frame.
- *
- * Note: For cache line sizes 256 or larger this value is going to end
- *	 up negative.  In these cases we should fall back to the 3K
- *	 buffers.
- */
-#if (PAGE_SIZE < 8192)
-#define IXGBE_MAX_2K_FRAME_BUILD_SKB (IXGBE_RXBUFFER_1536 - NET_IP_ALIGN)
-#define IXGBE_2K_TOO_SMALL_WITH_PADDING \
-((NET_SKB_PAD + IXGBE_RXBUFFER_1536) > SKB_WITH_OVERHEAD(IXGBE_RXBUFFER_2K))
-
-static inline int ixgbe_compute_pad(int rx_buf_len)
-{
-	int page_size, pad_size;
-
-	page_size = ALIGN(rx_buf_len, PAGE_SIZE / 2);
-	pad_size = SKB_WITH_OVERHEAD(page_size) - rx_buf_len;
-
-	return pad_size;
-}
-
-static inline int ixgbe_skb_pad(void)
-{
-	int rx_buf_len;
-
-	/* If a 2K buffer cannot handle a standard Ethernet frame then
-	 * optimize padding for a 3K buffer instead of a 1.5K buffer.
-	 *
-	 * For a 3K buffer we need to add enough padding to allow for
-	 * tailroom due to NET_IP_ALIGN possibly shifting us out of
-	 * cache-line alignment.
-	 */
-	if (IXGBE_2K_TOO_SMALL_WITH_PADDING)
-		rx_buf_len = IXGBE_RXBUFFER_3K + SKB_DATA_ALIGN(NET_IP_ALIGN);
-	else
-		rx_buf_len = IXGBE_RXBUFFER_1536;
-
-	/* if needed make room for NET_IP_ALIGN */
-	rx_buf_len -= NET_IP_ALIGN;
-
-	return ixgbe_compute_pad(rx_buf_len);
-}
-
-#define IXGBE_SKB_PAD	ixgbe_skb_pad()
-#else
-#define IXGBE_SKB_PAD	(NET_SKB_PAD + NET_IP_ALIGN)
-#endif
+#define IXGBE_RX_PAGE_LEN(hr)		(ALIGN_DOWN(LIBETH_RX_PAGE_LEN(hr), \
+					 IXGBE_SRRCTL_BSIZEPKT_STEP))
+#define IXGBE_RX_SRRCTL_BUF_SIZE(mtu)	(ALIGN((mtu) + LIBETH_RX_LL_LEN, \
+					 IXGBE_SRRCTL_BSIZEPKT_STEP))
 
 /*
  * NOTE: netdev_alloc_skb reserves up to 64 bytes, NET_IP_ALIGN means we
@@ -271,20 +222,9 @@ struct ixgbe_tx_buffer {
 	u32 tx_flags;
 };
 
-struct ixgbe_rx_buffer {
-	union {
-		struct {
-			struct sk_buff *skb;
-			dma_addr_t dma;
-			struct page *page;
-			__u32 page_offset;
-			__u16 pagecnt_bias;
-		};
-		struct {
-			bool discard;
-			struct xdp_buff *xdp;
-		};
-	};
+struct ixgbe_xsk_rx_buffer {
+	bool discard;
+	struct xdp_buff *xdp;
 };
 
 struct ixgbe_queue_stats {
@@ -311,8 +251,6 @@ struct ixgbe_rx_queue_stats {
 #define IXGBE_TS_HDR_LEN 8
 
 enum ixgbe_ring_state_t {
-	__IXGBE_RX_3K_BUFFER,
-	__IXGBE_RX_BUILD_SKB_ENABLED,
 	__IXGBE_RX_RSC_ENABLED,
 	__IXGBE_RX_CSUM_UDP_ZERO_ERR,
 	__IXGBE_RX_FCOE,
@@ -321,12 +259,10 @@ enum ixgbe_ring_state_t {
 	__IXGBE_TX_DETECT_HANG,
 	__IXGBE_HANG_CHECK_ARMED,
 	__IXGBE_TX_XDP_RING,
+	__IXGBE_TX_XDP_RING_PRIMED,
 	__IXGBE_TX_DISABLED,
 };
 
-#define ring_uses_build_skb(ring) \
-	test_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &(ring)->state)
-
 struct ixgbe_fwd_adapter {
 	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
 	struct net_device *netdev;
@@ -357,12 +293,22 @@ struct ixgbe_ring {
 	struct ixgbe_ring *next;	/* pointer to next ring in q_vector */
 	struct ixgbe_q_vector *q_vector; /* backpointer to host q_vector */
 	struct net_device *netdev;	/* netdev ring belongs to */
-	struct bpf_prog *xdp_prog;
-	struct device *dev;		/* device for DMA mapping */
+	struct bpf_prog __rcu *xdp_prog;
+	union {
+		struct page_pool *pp;	/* Rx and XDP rings */
+		struct device *dev;	/* Tx ring */
+	};
 	void *desc;			/* descriptor ring memory */
 	union {
+		u32 truesize;		/* Rx buffer full size */
+		u32 pending;		/* Sent-not-completed descriptors */
+	};
+	u32 hdr_truesize;		/* Rx header buffer full size */
+	union {
+		struct libeth_fqe *rx_fqes;
+		struct ixgbe_xsk_rx_buffer *rx_xsk_buffer_info;
 		struct ixgbe_tx_buffer *tx_buffer_info;
-		struct ixgbe_rx_buffer *rx_buffer_info;
+		struct libeth_sqe *xdp_sqes;
 	};
 	unsigned long state;
 	u8 __iomem *tail;
@@ -377,17 +323,14 @@ struct ixgbe_ring {
 					 * associated with this ring, which is
 					 * different for DCB and RSS modes
 					 */
-	u16 next_to_use;
 	u16 next_to_clean;
+	u32 next_to_use;
 
 	unsigned long last_rx_timestamp;
 
-	union {
-		u16 next_to_alloc;
-		struct {
-			u8 atr_sample_rate;
-			u8 atr_count;
-		};
+	struct {
+		u8 atr_sample_rate;
+		u8 atr_count;
 	};
 
 	u8 dcb_tc;
@@ -397,12 +340,17 @@ struct ixgbe_ring {
 		struct ixgbe_tx_queue_stats tx_stats;
 		struct ixgbe_rx_queue_stats rx_stats;
 	};
-	u16 rx_offset;
+	struct libeth_fqe *hdr_fqes;
+	struct page_pool *hdr_pp;
 	struct xdp_rxq_info xdp_rxq;
+	struct libeth_xdpsq_lock xdpq_lock;
+	u32 cached_ntu;
 	spinlock_t tx_lock;	/* used in XDP mode */
 	struct xsk_buff_pool *xsk_pool;
 	u16 ring_idx;		/* {rx,tx,xdp}_ring back reference idx */
-	u16 rx_buf_len;
+	u32 hdr_buf_len;
+	u32 rx_buf_len;
+	struct libeth_xdp_buff_stash xdp_stash;
 } ____cacheline_internodealigned_in_smp;
 
 enum ixgbe_ring_f_enum {
@@ -433,8 +381,6 @@ enum ixgbe_ring_f_enum {
 #define IXGBE_MAX_TX_DESCRIPTORS	40
 #define IXGBE_MAX_TX_VF_HANGS		4
 
-DECLARE_STATIC_KEY_FALSE(ixgbe_xdp_locking_key);
-
 struct ixgbe_ring_feature {
 	u16 limit;	/* upper limit on feature indices */
 	u16 indices;	/* current value of indices */
@@ -446,31 +392,6 @@ struct ixgbe_ring_feature {
 #define IXGBE_82599_VMDQ_4Q_MASK 0x7C
 #define IXGBE_82599_VMDQ_2Q_MASK 0x7E
 
-/*
- * FCoE requires that all Rx buffers be over 2200 bytes in length.  Since
- * this is twice the size of a half page we need to double the page order
- * for FCoE enabled Rx queues.
- */
-static inline unsigned int ixgbe_rx_bufsz(struct ixgbe_ring *ring)
-{
-	if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state))
-		return IXGBE_RXBUFFER_3K;
-#if (PAGE_SIZE < 8192)
-	if (ring_uses_build_skb(ring))
-		return IXGBE_MAX_2K_FRAME_BUILD_SKB;
-#endif
-	return IXGBE_RXBUFFER_2K;
-}
-
-static inline unsigned int ixgbe_rx_pg_order(struct ixgbe_ring *ring)
-{
-#if (PAGE_SIZE < 8192)
-	if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state))
-		return 1;
-#endif
-	return 0;
-}
-#define ixgbe_rx_pg_size(_ring) (PAGE_SIZE << ixgbe_rx_pg_order(_ring))
 
 #define IXGBE_ITR_ADAPTIVE_MIN_INC	2
 #define IXGBE_ITR_ADAPTIVE_MIN_USECS	10
@@ -672,7 +593,7 @@ struct ixgbe_adapter {
 #define IXGBE_FLAG2_VLAN_PROMISC		BIT(13)
 #define IXGBE_FLAG2_EEE_CAPABLE			BIT(14)
 #define IXGBE_FLAG2_EEE_ENABLED			BIT(15)
-#define IXGBE_FLAG2_RX_LEGACY			BIT(16)
+/* BIT16 used to be reserved for legacy RX flag */
 #define IXGBE_FLAG2_IPSEC_ENABLED		BIT(17)
 #define IXGBE_FLAG2_VF_IPSEC_ENABLED		BIT(18)
 #define IXGBE_FLAG2_AUTO_DISABLE_VF		BIT(19)
@@ -681,6 +602,7 @@ struct ixgbe_adapter {
 #define IXGBE_FLAG2_MOD_POWER_UNSUPPORTED	BIT(22)
 #define IXGBE_FLAG2_API_MISMATCH		BIT(23)
 #define IXGBE_FLAG2_FW_ROLLBACK			BIT(24)
+#define IXGBE_FLAG2_HSPLIT			BIT(25)
 
 	/* Tx fast path data */
 	int num_tx_queues;
@@ -859,10 +781,7 @@ static inline struct ixgbe_adapter *ixgbe_from_netdev(struct net_device *netdev)
 
 static inline int ixgbe_determine_xdp_q_idx(int cpu)
 {
-	if (static_key_enabled(&ixgbe_xdp_locking_key))
-		return cpu % IXGBE_MAX_XDP_QS;
-	else
-		return cpu;
+	return cpu % IXGBE_MAX_XDP_QS;
 }
 
 static inline
@@ -919,7 +838,6 @@ struct ixgbe_cb {
 	};
 	dma_addr_t dma;
 	u16 append_cnt;
-	bool page_released;
 };
 #define IXGBE_CB(skb) ((struct ixgbe_cb *)(skb)->cb)
 

drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c

@@ -132,11 +132,9 @@ static const char ixgbe_gstrings_test[][ETH_GSTRING_LEN] = {
 #define IXGBE_TEST_LEN sizeof(ixgbe_gstrings_test) / ETH_GSTRING_LEN
 
 static const char ixgbe_priv_flags_strings[][ETH_GSTRING_LEN] = {
-#define IXGBE_PRIV_FLAGS_LEGACY_RX	BIT(0)
-	"legacy-rx",
-#define IXGBE_PRIV_FLAGS_VF_IPSEC_EN	BIT(1)
+#define IXGBE_PRIV_FLAGS_VF_IPSEC_EN	BIT(0)
 	"vf-ipsec",
-#define IXGBE_PRIV_FLAGS_AUTO_DISABLE_VF	BIT(2)
+#define IXGBE_PRIV_FLAGS_AUTO_DISABLE_VF	BIT(1)
 	"mdd-disable-vf",
 };
 
@@ -1951,7 +1949,6 @@ static int ixgbe_setup_desc_rings(struct ixgbe_adapter *adapter)
 	/* Setup Rx Descriptor ring and Rx buffers */
 	rx_ring->count = IXGBE_DEFAULT_RXD;
 	rx_ring->queue_index = 0;
-	rx_ring->dev = &adapter->pdev->dev;
 	rx_ring->netdev = adapter->netdev;
 	rx_ring->reg_idx = adapter->rx_ring[0]->reg_idx;
 
@@ -2058,14 +2055,16 @@ static void ixgbe_create_lbtest_frame(struct sk_buff *skb,
 	skb->data[frame_size + 12] = 0xAF;
 }
 
-static bool ixgbe_check_lbtest_frame(struct ixgbe_rx_buffer *rx_buffer,
+static bool ixgbe_check_lbtest_frame(const struct libeth_fqe *rx_buffer,
 				     unsigned int frame_size)
 {
+	u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset;
 	unsigned char *data;
 
 	frame_size >>= 1;
 
-	data = page_address(rx_buffer->page) + rx_buffer->page_offset;
+	data = page_address(__netmem_to_page(rx_buffer->netmem)) +
+	       rx_buffer->offset + hr;
 
 	return data[3] == 0xFF && data[frame_size + 10] == 0xBE &&
 		data[frame_size + 12] == 0xAF;
@@ -2113,28 +2112,22 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring,
 	}
 
 	while (rx_desc->wb.upper.length) {
-		struct ixgbe_rx_buffer *rx_buffer;
+		struct libeth_fqe *rx_buffer;
 
 		/* check Rx buffer */
-		rx_buffer = &rx_ring->rx_buffer_info[rx_ntc];
+		rx_buffer = &rx_ring->rx_fqes[rx_ntc];
 
 		/* sync Rx buffer for CPU read */
-		dma_sync_single_for_cpu(rx_ring->dev,
-					rx_buffer->dma,
-					ixgbe_rx_bufsz(rx_ring),
-					DMA_FROM_DEVICE);
+		libeth_rx_sync_for_cpu(rx_buffer, rx_ring->rx_buf_len);
 
 		/* verify contents of skb */
 		if (ixgbe_check_lbtest_frame(rx_buffer, size))
 			count++;
 		else
 			break;
 
-		/* sync Rx buffer for device write */
-		dma_sync_single_for_device(rx_ring->dev,
-					   rx_buffer->dma,
-					   ixgbe_rx_bufsz(rx_ring),
-					   DMA_FROM_DEVICE);
+		/* recycle the page back to the pool */
+		libeth_rx_recycle_slow(rx_buffer->netmem);
 
 		/* increment Rx next to clean counter */
 		rx_ntc++;
@@ -3667,9 +3660,6 @@ static u32 ixgbe_get_priv_flags(struct net_device *netdev)
 	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
 	u32 priv_flags = 0;
 
-	if (adapter->flags2 & IXGBE_FLAG2_RX_LEGACY)
-		priv_flags |= IXGBE_PRIV_FLAGS_LEGACY_RX;
-
 	if (adapter->flags2 & IXGBE_FLAG2_VF_IPSEC_ENABLED)
 		priv_flags |= IXGBE_PRIV_FLAGS_VF_IPSEC_EN;
 
@@ -3685,10 +3675,6 @@ static int ixgbe_set_priv_flags(struct net_device *netdev, u32 priv_flags)
 	unsigned int flags2 = adapter->flags2;
 	unsigned int i;
 
-	flags2 &= ~IXGBE_FLAG2_RX_LEGACY;
-	if (priv_flags & IXGBE_PRIV_FLAGS_LEGACY_RX)
-		flags2 |= IXGBE_FLAG2_RX_LEGACY;
-
 	flags2 &= ~IXGBE_FLAG2_VF_IPSEC_ENABLED;
 	if (priv_flags & IXGBE_PRIV_FLAGS_VF_IPSEC_EN)
 		flags2 |= IXGBE_FLAG2_VF_IPSEC_ENABLED;