diff --git a/drivers/net/ethernet/intel/ixgbevf/Makefile b/drivers/net/ethernet/intel/ixgbevf/Makefile
index 01d3e892f3fa7b..cdae62f25fd926 100644
--- a/drivers/net/ethernet/intel/ixgbevf/Makefile
+++ b/drivers/net/ethernet/intel/ixgbevf/Makefile
@@ -6,5 +6,5 @@
 
 obj-$(CONFIG_IXGBEVF) += ixgbevf.o
 
-ixgbevf-y := vf.o mbx.o ethtool.o ixgbevf_main.o
+ixgbevf-y := vf.o mbx.o ethtool.o ixgbevf_main.o ixgbevf_xsk.o
 ixgbevf-$(CONFIG_IXGBEVF_IPSEC) += ipsec.o
diff --git a/drivers/net/ethernet/intel/ixgbevf/ethtool.c b/drivers/net/ethernet/intel/ixgbevf/ethtool.c
index 274eef39c58618..387eae14ed44d7 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ethtool.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ethtool.c
@@ -916,6 +916,75 @@ static int ixgbevf_get_rxfh(struct net_device *netdev,
 	return err;
 }
 
+static void ixgbevf_get_reported_q_num(u32 rx, u32 tx, u32 *dst_combined,
+				       u32 *dst_rx, u32 *dst_tx)
+{
+	rx = min_t(u32, rx, num_online_cpus());
+	*dst_combined = min_t(u32, rx, tx);
+	if (rx > tx) {
+		*dst_rx = rx - tx;
+		*dst_tx = 0;
+	} else {
+		*dst_tx = tx - rx;
+		*dst_rx = 0;
+	}
+}
+
+static void ixgbevf_get_channels(struct net_device *netdev,
+				 struct ethtool_channels *ch)
+{
+	struct ixgbevf_adapter *adapter = netdev_priv(netdev);
+
+	ixgbevf_get_reported_q_num(adapter->num_rx_queues,
+				   adapter->num_tx_queues,
+				   &ch->combined_count, &ch->rx_count,
+				   &ch->tx_count);
+
+	ixgbevf_get_reported_q_num(adapter->q_caps.max_rxqs,
+				   adapter->q_caps.max_txqs,
+				   &ch->max_combined, &ch->max_rx,
+				   &ch->max_tx);
+
+	ch->max_other = NON_Q_VECTORS;
+	ch->other_count = NON_Q_VECTORS;
+}
+
+static int ixgbevf_set_channels(struct net_device *netdev,
+				struct ethtool_channels *ch)
+{
+	struct ixgbevf_adapter *adapter = netdev_priv(netdev);
+	u32 num_req = ch->combined_count;
+	bool was_up;
+	int err = 0;
+
+	/* Do not change queue number if DCB is enabled */
+	if (adapter->q_caps.num_tcs > 1)
+		return -EOPNOTSUPP;
+
+	if (ch->rx_count || ch->tx_count || ch->other_count != NON_Q_VECTORS)
+		return -EINVAL;
+
+	if (num_req == adapter->num_rx_queues &&
+	    num_req == adapter->num_tx_queues)
+		return 0;
+
+	adapter->num_req_qpairs = num_req;
+
+	was_up = netif_running(netdev);
+	if (was_up)
+		ixgbevf_close(netdev);
+
+	ixgbevf_clear_interrupt_scheme(adapter);
+	err = ixgbevf_init_interrupt_scheme(adapter);
+	if (err)
+		return err;
+
+	if (was_up)
+		ixgbevf_open(netdev);
+
+	return 0;
+}
+
 static const struct ethtool_ops ixgbevf_ethtool_ops = {
 	.supported_coalesce_params = ETHTOOL_COALESCE_USECS,
 	.get_drvinfo		= ixgbevf_get_drvinfo,
@@ -938,6 +1007,8 @@ static const struct ethtool_ops ixgbevf_ethtool_ops = {
 	.get_rxfh_key_size	= ixgbevf_get_rxfh_key_size,
 	.get_rxfh		= ixgbevf_get_rxfh,
 	.get_link_ksettings	= ixgbevf_get_link_ksettings,
+	.get_channels		= ixgbevf_get_channels,
+	.set_channels		= ixgbevf_set_channels,
 };
 
 void ixgbevf_set_ethtool_ops(struct net_device *netdev)
diff --git a/drivers/net/ethernet/intel/ixgbevf/ipsec.c b/drivers/net/ethernet/intel/ixgbevf/ipsec.c
index fce35924ff8b51..706e123fa92284 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ipsec.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ipsec.c
@@ -450,11 +450,11 @@ static const struct xfrmdev_ops ixgbevf_xfrmdev_ops = {
  * @first: current data packet
  * @itd: ipsec Tx data for later use in building context descriptor
  **/
-int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring,
-		     struct ixgbevf_tx_buffer *first,
+int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring, struct libeth_sqe *first,
 		     struct ixgbevf_ipsec_tx_data *itd)
 {
 	struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev);
+	struct ixgbevf_skb_sqe_priv *sqe_priv = (void *)&first->priv;
 	struct ixgbevf_ipsec *ipsec = adapter->ipsec;
 	struct xfrm_state *xs;
 	struct sec_path *sp;
@@ -491,12 +491,12 @@ int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring,
 
 	itd->pfsa = tsa->pfsa - IXGBE_IPSEC_BASE_TX_INDEX;
 
-	first->tx_flags |= IXGBE_TX_FLAGS_IPSEC | IXGBE_TX_FLAGS_CSUM;
+	sqe_priv->tx_flags |= IXGBE_TX_FLAGS_IPSEC | IXGBE_TX_FLAGS_CSUM;
 
 	if (xs->id.proto == IPPROTO_ESP) {
 		itd->flags |= IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP |
 			      IXGBE_ADVTXD_TUCMD_L4T_TCP;
-		if (first->protocol == htons(ETH_P_IP))
+		if (sqe_priv->protocol == htons(ETH_P_IP))
 			itd->flags |= IXGBE_ADVTXD_TUCMD_IPV4;
 
 		/* The actual trailer length is authlen (16 bytes) plus
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h
index 17958cfb4ee65b..ebdc3304e768eb 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h
@@ -11,7 +11,7 @@
 #include <linux/netdevice.h>
 #include <linux/if_vlan.h>
 #include <linux/u64_stats_sync.h>
-#include <net/libeth/types.h>
+#include <net/libeth/tx.h>
 #include <net/xdp.h>
 
 #include "vf.h"
@@ -24,24 +24,12 @@
 #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)
 #define DESC_NEEDED (MAX_SKB_FRAGS + 4)
 
-/* wrapper around a pointer to a socket buffer,
- * so a DMA handle can be stored along with the buffer
- */
-struct ixgbevf_tx_buffer {
-	union ixgbe_adv_tx_desc *next_to_watch;
-	unsigned long time_stamp;
-	union {
-		struct sk_buff *skb;
-		/* XDP uses address ptr on irq_clean */
-		void *data;
-	};
-	unsigned int bytecount;
-	unsigned short gso_segs;
-	__be16 protocol;
-	DEFINE_DMA_UNMAP_ADDR(dma);
-	DEFINE_DMA_UNMAP_LEN(len);
+struct ixgbevf_skb_sqe_priv {
 	u32 tx_flags;
+	__be16 protocol;
 };
+static_assert(sizeof(struct ixgbevf_skb_sqe_priv) <=
+	      sizeof(typeof_member(struct libeth_sqe, priv)));
 
 struct ixgbevf_stats {
 	u64 packets;
@@ -66,6 +54,7 @@ enum ixgbevf_ring_state_t {
 	__IXGBEVF_HANG_CHECK_ARMED,
 	__IXGBEVF_TX_XDP_RING,
 	__IXGBEVF_TX_XDP_RING_PRIMED,
+	__IXGBEVF_RXTX_XSK_RING,
 };
 
 #define ring_is_xdp(ring) \
@@ -75,6 +64,13 @@ enum ixgbevf_ring_state_t {
 #define clear_ring_xdp(ring) \
 		clear_bit(__IXGBEVF_TX_XDP_RING, &(ring)->state)
 
+#define ring_is_xsk(ring) \
+		test_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state)
+#define set_ring_xsk(ring) \
+		set_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state)
+#define clear_ring_xsk(ring) \
+		clear_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state)
+
 struct ixgbevf_ring {
 	struct ixgbevf_ring *next;
 	struct ixgbevf_q_vector *q_vector;	/* backpointer to q_vector */
@@ -85,22 +81,22 @@ struct ixgbevf_ring {
 		struct device *dev;	/* Tx ring */
 	};
 	void *desc;			/* descriptor ring memory */
-	union {
-		u32 truesize;		/* Rx buffer full size */
-		u32 pending;		/* Sent-not-completed descriptors */
-	};
+	u32 truesize;			/* Rx buffer full size */
 	u32 hdr_truesize;		/* Rx header buffer full size */
 	u16 count;			/* amount of descriptors */
 	u16 next_to_clean;
 	u32 next_to_use;
+	u32 pending;			/* Sent-not-completed descriptors */
 
 	union {
 		struct libeth_fqe *rx_fqes;
-		struct ixgbevf_tx_buffer *tx_buffer_info;
+		struct libeth_xdp_buff	**xsk_fqes;
+		struct libeth_sqe *tx_sqes;
 		struct libeth_sqe *xdp_sqes;
 	};
 	struct libeth_xdpsq_lock xdpq_lock;
 	u32 cached_ntu;
+	u32 thresh;
 	unsigned long state;
 	struct ixgbevf_stats stats;
 	struct u64_stats_sync syncp;
@@ -121,8 +117,10 @@ struct ixgbevf_ring {
 	int queue_index; /* needed for multiqueue queue management */
 	u32 rx_buf_len;
 	struct libeth_xdp_buff_stash xdp_stash;
+	struct libeth_xdp_buff *xsk_xdp_head;
 	unsigned int dma_size;		/* length in bytes */
 	dma_addr_t dma;			/* phys. address of descriptor ring */
+	struct xsk_buff_pool *xsk_pool; /* AF_XDP ZC rings */
 } ____cacheline_internodealigned_in_smp;
 
 /* How many Rx Buffers do we bundle into one write to the hardware ? */
@@ -235,6 +233,14 @@ static inline u16 ixgbevf_desc_unused(struct ixgbevf_ring *ring)
 	return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - 1;
 }
 
+static inline u16 ixgbevf_desc_used(struct ixgbevf_ring *ring)
+{
+	u16 ntc = ring->next_to_clean;
+	u16 ntu = ring->next_to_use;
+
+	return ((ntu >= ntc) ? 0 : ring->count) + ntu - ntc;
+}
+
 static inline void ixgbevf_write_tail(struct ixgbevf_ring *ring, u32 value)
 {
 	writel(value, ring->tail);
@@ -260,6 +266,15 @@ static inline void ixgbevf_write_tail(struct ixgbevf_ring *ring, u32 value)
 #define IXGBEVF_RX_DMA_ATTR \
 	(DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
 
+struct ixgbevf_q_caps {
+	u32 min_rxqs;
+	u32 max_rxqs;
+	u32 min_txqs;
+	u32 max_txqs;
+	u32 num_tcs;
+	u32 def_tc;
+};
+
 /* board specific private data structure */
 struct ixgbevf_adapter {
 	/* this field must be first, see ixgbevf_process_skb_fields */
@@ -336,6 +351,8 @@ struct ixgbevf_adapter {
 	u8 rss_indir_tbl[IXGBEVF_X550_VFRETA_SIZE];
 	u32 flags;
 	bool link_state;
+	struct ixgbevf_q_caps q_caps;
+	u32 num_req_qpairs;
 
 #ifdef CONFIG_XFRM
 	struct ixgbevf_ipsec *ipsec;
@@ -399,15 +416,31 @@ int ixgbevf_open(struct net_device *netdev);
 int ixgbevf_close(struct net_device *netdev);
 void ixgbevf_up(struct ixgbevf_adapter *adapter);
 void ixgbevf_down(struct ixgbevf_adapter *adapter);
+void ixgbevf_flush_tx_queue(struct ixgbevf_ring *ring);
+void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter,
+			      struct ixgbevf_ring *ring);
+void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
+				  struct ixgbevf_ring *ring);
 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter);
 void ixgbevf_reset(struct ixgbevf_adapter *adapter);
 void ixgbevf_set_ethtool_ops(struct net_device *netdev);
 int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter,
 			       struct ixgbevf_ring *rx_ring);
+void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter);
+void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
+			       struct ixgbevf_ring *ring);
 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *);
+void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
+			       struct ixgbevf_ring *ring);
 void ixgbevf_free_rx_resources(struct ixgbevf_ring *);
+void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring);
+void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring);
 void ixgbevf_free_tx_resources(struct ixgbevf_ring *);
+void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring);
+void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring);
 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter);
+int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter);
+void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter);
 int ethtool_ioctl(struct ifreq *ifr);
 
 extern void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector);
@@ -419,8 +452,7 @@ void ixgbevf_ipsec_restore(struct ixgbevf_adapter *adapter);
 void ixgbevf_ipsec_rx(struct ixgbevf_ring *rx_ring,
 		      union ixgbe_adv_rx_desc *rx_desc,
 		      struct sk_buff *skb);
-int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring,
-		     struct ixgbevf_tx_buffer *first,
+int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring, struct libeth_sqe *first,
 		     struct ixgbevf_ipsec_tx_data *itd);
 #else
 static inline void ixgbevf_init_ipsec_offload(struct ixgbevf_adapter *adapter)
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
index f301ded0067bc4..1a386b345978bd 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
@@ -30,11 +30,12 @@
 #include <linux/bpf.h>
 #include <linux/bpf_trace.h>
 #include <linux/atomic.h>
-#include <net/libeth/xdp.h>
+#include <net/libeth/xsk.h>
 #include <net/xfrm.h>
 
 #include "ixgbevf_txrx_lib.h"
 #include "ixgbevf_xdp_lib.h"
+#include "ixgbevf_xsk.h"
 
 const char ixgbevf_driver_name[] = "ixgbevf";
 static const char ixgbevf_driver_string[] =
@@ -253,13 +254,58 @@ static void ixgbevf_tx_timeout_reset(struct ixgbevf_adapter *adapter)
  * @netdev: network interface device structure
  * @txqueue: transmit queue hanging (unused)
  **/
-static void ixgbevf_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
+static void ixgbevf_tx_timeout(struct net_device *netdev, unsigned int txqueue)
 {
 	struct ixgbevf_adapter *adapter = netdev_priv(netdev);
+	struct ixgbevf_ring *tx_ring = adapter->tx_ring[txqueue];
+	u32 ntc = tx_ring->next_to_clean;
 
+	netdev_err(netdev, "Timeout info:\n");
+	netdev_err(netdev, "ntu=%u, ntc=%u, rs_idx=%u, cached_ntu=%u\n", tx_ring->next_to_use, ntc, tx_ring->tx_sqes[ntc].rs_idx, tx_ring->cached_ntu);
 	ixgbevf_tx_timeout_reset(adapter);
 }
 
+static u16 ixgbevf_clean_tx_num(struct ixgbevf_ring *tx_ring, bool in_napi,
+				 u16 to_clean,
+				 struct libeth_sq_napi_stats *stats,
+				 u16 *total_ipsec)
+{
+	bool xsk_ring = ring_is_xsk(tx_ring);
+	u16 ntc = tx_ring->next_to_clean;
+	struct xdp_frame_bulk xdp_bulk;
+	struct libeth_cq_pp cq = {
+		.ss = stats,
+		.dev = tx_ring->dev,
+		.napi = in_napi,
+		.bq = &xdp_bulk,
+	};
+	u32 xsk_frames = 0;
+
+	xdp_frame_bulk_init(&xdp_bulk);
+	for (int i = 0; i < to_clean; i++) {
+		struct ixgbevf_skb_sqe_priv *priv;
+		struct libeth_sqe *sqe;
+
+		sqe = &tx_ring->tx_sqes[ntc];
+		priv = (void *)&sqe->priv;
+		if (priv->tx_flags & IXGBE_TX_FLAGS_IPSEC &&
+		    likely(total_ipsec))
+			(*total_ipsec)++;
+
+		xsk_frames += xsk_ring && !sqe->type ? 1 : 0;
+		libeth_tx_complete_any(sqe, &cq);
+
+		if (unlikely(++ntc == tx_ring->count))
+			ntc = 0;
+	}
+
+	xdp_flush_frame_bulk(&xdp_bulk);
+	if (xsk_frames)
+		xsk_tx_completed(tx_ring->xsk_pool, xsk_frames);
+
+	return ntc;
+}
+
 /**
  * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
  * @q_vector: board private structure
@@ -270,107 +316,36 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
 				 struct ixgbevf_ring *tx_ring, int napi_budget)
 {
 	struct ixgbevf_adapter *adapter = q_vector->adapter;
-	struct ixgbevf_tx_buffer *tx_buffer;
-	union ixgbe_adv_tx_desc *tx_desc;
-	unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0;
-	unsigned int budget = tx_ring->count / 2;
-	unsigned int i = tx_ring->next_to_clean;
+	u16 budget = tx_ring->count / 2, to_clean, ntc;
+	struct libeth_sq_napi_stats stats = { };
+	u16 total_ipsec;
 
 	if (test_bit(__IXGBEVF_DOWN, &adapter->state))
 		return true;
 
-	tx_buffer = &tx_ring->tx_buffer_info[i];
-	tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
-	i -= tx_ring->count;
+	to_clean = ixgbevf_desc_used(tx_ring);
+	to_clean = ixgbevf_tx_get_num_sent(tx_ring, min_t(u16, budget, to_clean));
+	budget = budget > to_clean ? budget - to_clean : 0;
+	ntc = ixgbevf_clean_tx_num(tx_ring, true, to_clean, &stats, &total_ipsec);
 
-	do {
-		union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
-
-		/* if next_to_watch is not set then there is no work pending */
-		if (!eop_desc)
-			break;
-
-		/* prevent any other reads prior to eop_desc */
-		smp_rmb();
-
-		/* if DD is not set pending work has not been completed */
-		if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
-			break;
+	smp_wmb();
 
-		/* clear next_to_watch to prevent false hangs */
-		tx_buffer->next_to_watch = NULL;
-
-		/* update the statistics for this packet */
-		total_bytes += tx_buffer->bytecount;
-		total_packets += tx_buffer->gso_segs;
-		if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_IPSEC)
-			total_ipsec++;
-
-		/* free the skb */
-		napi_consume_skb(tx_buffer->skb, napi_budget);
-
-		/* unmap skb header data */
-		dma_unmap_single(tx_ring->dev,
-				 dma_unmap_addr(tx_buffer, dma),
-				 dma_unmap_len(tx_buffer, len),
-				 DMA_TO_DEVICE);
-
-		/* clear tx_buffer data */
-		dma_unmap_len_set(tx_buffer, len, 0);
-
-		/* unmap remaining buffers */
-		while (tx_desc != eop_desc) {
-			tx_buffer++;
-			tx_desc++;
-			i++;
-			if (unlikely(!i)) {
-				i -= tx_ring->count;
-				tx_buffer = tx_ring->tx_buffer_info;
-				tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
-			}
-
-			/* unmap any remaining paged data */
-			if (dma_unmap_len(tx_buffer, len)) {
-				dma_unmap_page(tx_ring->dev,
-					       dma_unmap_addr(tx_buffer, dma),
-					       dma_unmap_len(tx_buffer, len),
-					       DMA_TO_DEVICE);
-				dma_unmap_len_set(tx_buffer, len, 0);
-			}
-		}
-
-		/* move us one more past the eop_desc for start of next pkt */
-		tx_buffer++;
-		tx_desc++;
-		i++;
-		if (unlikely(!i)) {
-			i -= tx_ring->count;
-			tx_buffer = tx_ring->tx_buffer_info;
-			tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
-		}
-
-		/* issue prefetch for next Tx descriptor */
-		prefetch(tx_desc);
-
-		/* update budget accounting */
-		budget--;
-	} while (likely(budget));
-
-	i += tx_ring->count;
-	tx_ring->next_to_clean = i;
+	tx_ring->next_to_clean = ntc;
 	u64_stats_update_begin(&tx_ring->syncp);
-	tx_ring->stats.bytes += total_bytes;
-	tx_ring->stats.packets += total_packets;
+	tx_ring->stats.bytes += stats.bytes;
+	tx_ring->stats.packets += stats.packets;
 	u64_stats_update_end(&tx_ring->syncp);
-	q_vector->tx.total_bytes += total_bytes;
-	q_vector->tx.total_packets += total_packets;
+	q_vector->tx.total_bytes += stats.bytes;
+	q_vector->tx.total_packets += stats.packets;
 	adapter->tx_ipsec += total_ipsec;
 
 	if (check_for_tx_hang(tx_ring) && ixgbevf_check_tx_hang(tx_ring)) {
 		struct ixgbe_hw *hw = &adapter->hw;
 		union ixgbe_adv_tx_desc *eop_desc;
+		u32 rs_idx;
 
-		eop_desc = tx_ring->tx_buffer_info[i].next_to_watch;
+		rs_idx = tx_ring->tx_sqes[ntc].rs_idx;
+		eop_desc = rs_idx ? IXGBEVF_TX_DESC(tx_ring, rs_idx - 1) : NULL;
 
 		pr_err("Detected Tx Unit Hang%s\n"
 		       "  Tx Queue             <%d>\n"
@@ -379,16 +354,13 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
 		       "  next_to_clean        <%x>\n"
 		       "tx_buffer_info[next_to_clean]\n"
 		       "  next_to_watch        <%p>\n"
-		       "  eop_desc->wb.status  <%x>\n"
-		       "  time_stamp           <%lx>\n"
-		       "  jiffies              <%lx>\n",
+		       "  eop_desc->wb.status  <%x>\n",
 		       ring_is_xdp(tx_ring) ? " XDP" : "",
 		       tx_ring->queue_index,
 		       IXGBE_READ_REG(hw, IXGBE_VFTDH(tx_ring->reg_idx)),
 		       IXGBE_READ_REG(hw, IXGBE_VFTDT(tx_ring->reg_idx)),
-		       tx_ring->next_to_use, i,
-		       eop_desc, (eop_desc ? eop_desc->wb.status : 0),
-		       tx_ring->tx_buffer_info[i].time_stamp, jiffies);
+		       tx_ring->next_to_use, ntc,
+		       eop_desc, (eop_desc ? eop_desc->wb.status : 0));
 
 		netif_stop_subqueue(tx_ring->netdev,
 				    tx_ring->queue_index);
@@ -400,7 +372,7 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
 	}
 
 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
-	if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
+	if (unlikely(stats.packets && netif_carrier_ok(tx_ring->netdev) &&
 		     (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
 		/* Make sure that anybody stopping the queue after this
 		 * sees the new next_to_clean.
@@ -500,7 +472,7 @@ static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
 	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
 }
 
-static void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring)
+void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring)
 {
 	ixgbevf_clean_xdp_num(xdp_ring, false, xdp_ring->pending);
 	libeth_xdpsq_put(&xdp_ring->xdpq_lock, xdp_ring->netdev);
@@ -545,11 +517,30 @@ static void ixgbevf_xdp_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i,
 LIBETH_XDP_DEFINE_START();
 LIBETH_XDP_DEFINE_FLUSH_TX(static ixgbevf_xdp_flush_tx, ixgbevf_prep_xdp_sq,
 			   ixgbevf_xdp_xmit_desc);
+__LIBETH_XDP_DEFINE_FLUSH_TX(static ixgbevf_xdp_flush_tx_shared,
+			     ixgbevf_prep_tx_sq, ixgbevf_xdp_xmit_desc,
+			     ixgbevf_xdp_tx_unprep, xdp);
 LIBETH_XDP_DEFINE_FLUSH_XMIT(static ixgbevf_xdp_flush_xmit, ixgbevf_prep_xdp_sq,
 			     ixgbevf_xdp_xmit_desc);
-LIBETH_XDP_DEFINE_RUN_PROG(static ixgbevf_xdp_run_prog, ixgbevf_xdp_flush_tx);
+__LIBETH_XDP_DEFINE_FLUSH_XMIT(static ixgbevf_xdp_flush_xmit_shared,
+			       ixgbevf_prep_tx_sq, ixgbevf_xdp_xmit_desc,
+			       ixgbevf_xdp_tx_unprep);
+LIBETH_XDP_DEFINE_END();
+
+static bool ixgbevf_xdp_flush_common(struct libeth_xdp_tx_bulk *bq, u32 flags)
+{
+	const struct ixgbevf_ring *ring = bq->xdpsq;
+
+	if (test_bit(__IXGBEVF_TX_XDP_RING, &ring->state))
+		return ixgbevf_xdp_flush_tx(bq, flags);
+	else
+		return ixgbevf_xdp_flush_tx_shared(bq, flags);
+}
+
+LIBETH_XDP_DEFINE_START();
+LIBETH_XDP_DEFINE_RUN_PROG(static ixgbevf_xdp_run_prog, ixgbevf_xdp_flush_common);
 LIBETH_XDP_DEFINE_FINALIZE(static ixgbevf_xdp_finalize_xdp_napi,
-			   ixgbevf_xdp_flush_tx, ixgbevf_xdp_rs_and_bump);
+			   ixgbevf_xdp_flush_common, ixgbevf_xdp_rs_and_bump);
 LIBETH_XDP_DEFINE_END();
 
 static u32 ixgbevf_rx_hsplit_wa(const struct libeth_fqe *hdr,
@@ -589,9 +580,15 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
 	LIBETH_XDP_ONSTACK_BUFF(xdp);
 
 	libeth_xdp_init_buff(xdp, &rx_ring->xdp_stash, &rx_ring->xdp_rxq);
-	libeth_xdp_tx_init_bulk(&xdp_tx_bulk, rx_ring->xdp_prog,
-				adapter->netdev, adapter->xdp_ring,
-				adapter->num_xdp_queues);
+	if (adapter->num_xdp_queues)
+		libeth_xdp_tx_init_bulk(&xdp_tx_bulk, rx_ring->xdp_prog,
+					adapter->netdev, adapter->xdp_ring,
+					adapter->num_xdp_queues);
+	else
+		libeth_xdp_tx_init_bulk_shared(&xdp_tx_bulk, rx_ring->xdp_prog,
+					       adapter->netdev,
+					       adapter->tx_ring,
+					       adapter->num_tx_queues);
 
 	while (likely(total_rx_packets < budget)) {
 		union ixgbe_adv_rx_desc *rx_desc;
@@ -705,13 +702,26 @@ static int ixgbevf_xdp_xmit(struct net_device *dev, int n,
 	if (unlikely(test_bit(__IXGBEVF_DOWN, &adapter->state)))
 		return -ENETDOWN;
 
-	if (unlikely(!adapter->num_xdp_queues))
+	if (unlikely(!READ_ONCE(adapter->xdp_prog)))
 		return -ENXIO;
 
-	return libeth_xdp_xmit_do_bulk(dev, n, frames, flags, adapter->xdp_ring,
-				       adapter->num_xdp_queues,
-				       ixgbevf_xdp_flush_xmit,
-				       ixgbevf_xdp_rs_and_bump);
+	if (adapter->num_xdp_queues)
+		return libeth_xdp_xmit_do_bulk(dev, n, frames, flags,
+					       adapter->xdp_ring,
+					       adapter->num_xdp_queues,
+					       ixgbevf_xdp_flush_xmit,
+					       ixgbevf_xdp_rs_and_bump);
+	else {
+		LIBETH_XDP_ONSTACK_BULK(xdp_bulk);
+
+		libeth_xdp_xmit_init_bulk_shared(&xdp_bulk, dev,
+						 adapter->tx_ring,
+						 adapter->num_tx_queues);
+
+		return __libeth_xdp_xmit_do_bulk(&xdp_bulk, frames, n, flags,
+						 ixgbevf_xdp_flush_xmit_shared,
+						 ixgbevf_xdp_rs_and_bump);
+	}
 }
 
 /**
@@ -732,10 +742,13 @@ static int ixgbevf_poll(struct napi_struct *napi, int budget)
 	bool clean_complete = true;
 
 	ixgbevf_for_each_ring(ring, q_vector->tx) {
-		if (ring_is_xdp(ring))
-			continue;
-		if (!ixgbevf_clean_tx_irq(q_vector, ring, budget))
-			clean_complete = false;
+		if (!ring_is_xdp(ring))
+			clean_complete &=
+				ixgbevf_clean_tx_irq(q_vector, ring, budget);
+		if (ring_is_xsk(ring))
+			clean_complete &=
+				ixgbevf_clean_xsk_tx_irq(q_vector, ring,
+							 budget);
 	}
 
 	if (budget <= 0)
@@ -750,7 +763,10 @@ static int ixgbevf_poll(struct napi_struct *napi, int budget)
 		per_ring_budget = budget;
 
 	ixgbevf_for_each_ring(ring, q_vector->rx) {
-		int cleaned = ixgbevf_clean_rx_irq(q_vector, ring,
+		int cleaned = ring_is_xsk(ring) ?
+			      ixgbevf_clean_xsk_rx_irq(q_vector, ring,
+						       per_ring_budget) :
+			      ixgbevf_clean_rx_irq(q_vector, ring,
 						   per_ring_budget);
 		work_done += cleaned;
 		if (cleaned >= per_ring_budget)
@@ -1122,7 +1138,7 @@ static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
  * ixgbevf_irq_enable - Enable default interrupt generation settings
  * @adapter: board private structure
  **/
-static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
+void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 
@@ -1131,6 +1147,26 @@ static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
 	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
 }
 
+/**
+ * ixgbevf_xsk_pool_from_q - get ZC XSK buffer pool bound to a queue ID
+ * @ring: Rx or Tx ring
+ *
+ * Return: A pointer to xsk_buff_pool structure if there is a buffer pool
+ * attached, configured as zero-copy, and usable by this queue, NULL otherwise.
+ */
+static struct xsk_buff_pool *ixgbevf_xsk_pool_from_q(struct ixgbevf_ring *ring)
+{
+	struct ixgbevf_adapter *adapter = ring->q_vector->adapter;
+	struct xsk_buff_pool *pool =
+		xsk_get_pool_from_qid(ring->netdev, ring->queue_index);
+
+	if (!rcu_dereference(ring->xdp_prog) && !ring_is_xdp(ring) &&
+	    !(adapter->xdp_prog && !adapter->num_xdp_queues))
+		return NULL;
+
+	return (pool && pool->dev) ? pool : NULL;
+}
+
 /**
  * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset
  * @adapter: board private structure
@@ -1138,8 +1174,8 @@ static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
  *
  * Configure the Tx descriptor ring after a reset.
  **/
-static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
-				      struct ixgbevf_ring *ring)
+void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
+			       struct ixgbevf_ring *ring)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	u64 tdba = ring->dma;
@@ -1187,15 +1223,20 @@ static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
 		   32;           /* PTHRESH = 32 */
 
 	/* reinitialize tx_buffer_info */
-	if (!ring_is_xdp(ring)) {
-		memset(ring->tx_buffer_info, 0,
-		       sizeof(struct ixgbevf_tx_buffer) * ring->count);
-	} else {
-		memset(ring->xdp_sqes, 0,
-		       sizeof(struct libeth_sqe) * ring->count);
+	memset(ring->xdp_sqes, 0, sizeof(struct libeth_sqe) * ring->count);
+	if (ring_is_xdp(ring))
 		libeth_xdpsq_get(&ring->xdpq_lock, ring->netdev,
 				 num_possible_cpus() > adapter->num_xdp_queues);
-	}
+
+	ring->xsk_pool = ixgbevf_xsk_pool_from_q(ring);
+	if (adapter->xdp_prog &&
+	    (!adapter->num_xdp_queues || ring_is_xdp(ring)) && ring->xsk_pool)
+		set_ring_xsk(ring);
+	else
+		clear_ring_xsk(ring);
+
+	ring->thresh = ring_is_xsk(ring) ? IXGBEVF_XSK_TX_CLEAN_THRESH(ring) :
+					   XDP_BULK_QUEUE_SIZE;
 
 	clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state);
 	clear_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state);
@@ -1266,8 +1307,8 @@ static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter)
 }
 
 #define IXGBEVF_MAX_RX_DESC_POLL 10
-static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter,
-				     struct ixgbevf_ring *ring)
+void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter,
+			      struct ixgbevf_ring *ring)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
@@ -1291,10 +1332,15 @@ static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter,
 	if (!wait_loop)
 		pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n",
 		       reg_idx);
+
+	/* Specification calls for 100 usec of delay after
+	 * RXDCTL.ENABLE is cleared
+	 */
+	usleep_range(100, 200);
 }
 
-static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
-					 struct ixgbevf_ring *ring)
+void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
+				  struct ixgbevf_ring *ring)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
@@ -1370,14 +1416,14 @@ static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter)
 	IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc);
 }
 
-static void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring)
+void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring)
 {
 	struct libeth_fq fq = {
 		.pp	= rx_ring->pp,
 		.fqes	= rx_ring->rx_fqes,
 	};
 
-	if (!fq.pp)
+	if (!fq.pp && !rx_ring->xsk_fqes)
 		return;
 
 	if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) {
@@ -1385,6 +1431,21 @@ static void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring)
 		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 	}
 
+	if (test_and_clear_bit(__IXGBEVF_RXTX_XSK_RING, &rx_ring->state)) {
+		struct libeth_xskfq xskfq = {
+			.fqes = rx_ring->xsk_fqes,
+		};
+
+		libeth_xskfq_destroy(&xskfq);
+		rx_ring->xsk_fqes = NULL;
+		rx_ring->pending = xskfq.pending;
+		rx_ring->thresh = xskfq.thresh;
+		rx_ring->rx_buf_len = xskfq.buf_len;
+		rx_ring->xsk_pool = NULL;
+
+		return;
+	}
+
 	libeth_rx_fq_destroy(&fq);
 	rx_ring->rx_fqes = NULL;
 	rx_ring->pp = NULL;
@@ -1414,9 +1475,44 @@ static int ixgbevf_rx_create_pp(struct ixgbevf_ring *rx_ring)
 						      LIBETH_XDP_HEADROOM :
 						      LIBETH_SKB_HEADROOM),
 	};
+	struct xsk_buff_pool *pool;
 	u32 frame_size;
 	int ret;
 
+	pool = ixgbevf_xsk_pool_from_q(rx_ring);
+	if (pool) {
+		u32 frag_sz = xsk_pool_get_rx_frag_step(pool);
+		struct libeth_xskfq xskfq = {
+			.nid = numa_node_id(),
+			.count = rx_ring->count,
+			.pool = pool,
+		};
+
+		ret = libeth_xskfq_create(&xskfq);
+		if (ret)
+			return ret;
+
+		rx_ring->xsk_pool = xskfq.pool;
+		rx_ring->xsk_fqes = xskfq.fqes;
+		rx_ring->pending = xskfq.count - 1;
+		rx_ring->thresh = xskfq.thresh;
+		rx_ring->rx_buf_len = xskfq.buf_len;
+		set_ring_xsk(rx_ring);
+
+		ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev,
+					 rx_ring->queue_index, 0, frag_sz);
+		if (ret)
+			goto err;
+
+		ret = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq,
+						 MEM_TYPE_XSK_BUFF_POOL,
+						 rx_ring->xsk_pool);
+		if (ret)
+			goto err;
+
+		return 0;
+	}
+
 	/* Some HW requires DMA write sizes to be aligned to 1K,
 	 * which warrants fake header split usage, but this is
 	 * not an issue if the frame size is at its maximum of 3K
@@ -1467,8 +1563,8 @@ static int ixgbevf_rx_create_pp(struct ixgbevf_ring *rx_ring)
 	return ret;
 }
 
-static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
-				      struct ixgbevf_ring *ring)
+void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
+			       struct ixgbevf_ring *ring)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	union ixgbe_adv_rx_desc *rx_desc;
@@ -1509,6 +1605,7 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
 	/* reset ntu and ntc to place SW in sync with hardwdare */
 	ring->next_to_clean = 0;
 	ring->next_to_use = 0;
+	ring->pending = ixgbevf_desc_unused(ring);
 
 	err = ixgbevf_rx_create_pp(ring);
 	if (err) {
@@ -1526,7 +1623,8 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
 		rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | IXGBE_RXDCTL_RLPML_EN);
 		if (pkt_len <= IXGBE_RXDCTL_RLPMLMASK) {
 			rxdctl |= pkt_len | IXGBE_RXDCTL_RLPML_EN;
-			rlpml_valid = true;
+			if (pkt_len <= ring->rx_buf_len)
+				rlpml_valid = true;
 		}
 	}
 
@@ -1536,7 +1634,11 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
 	IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
 
 	ixgbevf_rx_desc_queue_enable(adapter, ring);
-	ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring));
+
+	if (ring_is_xsk(ring))
+		ixgbevf_xsk_alloc_rx_bufs(ring, ring->pending);
+	else
+		ixgbevf_alloc_rx_buffers(ring, ring->pending);
 }
 
 /**
@@ -1719,47 +1821,31 @@ static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
 	}
 }
 
-static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter)
+static void ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter)
 {
-	struct ixgbe_hw *hw = &adapter->hw;
-	unsigned int def_q = 0;
-	unsigned int num_tcs = 0;
 	unsigned int num_rx_queues = adapter->num_rx_queues;
 	unsigned int num_tx_queues = adapter->num_tx_queues;
-	int err;
-
-	spin_lock_bh(&adapter->mbx_lock);
-
-	/* fetch queue configuration from the PF */
-	err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
-
-	spin_unlock_bh(&adapter->mbx_lock);
-
-	if (err)
-		return err;
 
-	if (num_tcs > 1) {
+	if (adapter->q_caps.num_tcs > 1) {
 		/* we need only one Tx queue */
 		num_tx_queues = 1;
 
 		/* update default Tx ring register index */
-		adapter->tx_ring[0]->reg_idx = def_q;
+		adapter->tx_ring[0]->reg_idx = adapter->q_caps.def_tc;
 
 		/* we need as many queues as traffic classes */
-		num_rx_queues = num_tcs;
+		num_rx_queues = adapter->q_caps.num_tcs;
 	}
 
 	/* if we have a bad config abort request queue reset */
 	if ((adapter->num_rx_queues != num_rx_queues) ||
 	    (adapter->num_tx_queues != num_tx_queues)) {
 		/* force mailbox timeout to prevent further messages */
-		hw->mbx.timeout = 0;
+		adapter->hw.mbx.timeout = 0;
 
 		/* wait for watchdog to come around and bail us out */
 		set_bit(__IXGBEVF_QUEUE_RESET_REQUESTED, &adapter->state);
 	}
-
-	return 0;
 }
 
 static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
@@ -1925,8 +2011,13 @@ void ixgbevf_up(struct ixgbevf_adapter *adapter)
  * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
  * @rx_ring: ring to free buffers from
  **/
-static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
+void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
 {
+	if (ring_is_xsk(rx_ring)) {
+		ixgbevf_rx_xsk_ring_free_buffs(rx_ring);
+		goto reset;
+	}
+
 	/* Free Rx ring sk_buff */
 	libeth_xdp_return_stash(&rx_ring->xdp_stash);
 
@@ -1944,62 +2035,22 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
 			i = 0;
 	}
 
+reset:
 	rx_ring->next_to_clean = 0;
 	rx_ring->next_to_use = 0;
+	rx_ring->pending = 0;
 }
 
 /**
  * ixgbevf_clean_tx_ring - Free Tx Buffers
  * @tx_ring: ring to be cleaned
  **/
-static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring)
+void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring)
 {
-	u16 i = tx_ring->next_to_clean;
-	struct ixgbevf_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
-
-	while (i != tx_ring->next_to_use) {
-		union ixgbe_adv_tx_desc *eop_desc, *tx_desc;
-
-		/* Free all the Tx ring sk_buffs */
-		dev_kfree_skb_any(tx_buffer->skb);
-
-		/* unmap skb header data */
-		dma_unmap_single(tx_ring->dev,
-				 dma_unmap_addr(tx_buffer, dma),
-				 dma_unmap_len(tx_buffer, len),
-				 DMA_TO_DEVICE);
-
-		/* check for eop_desc to determine the end of the packet */
-		eop_desc = tx_buffer->next_to_watch;
-		tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
-
-		/* unmap remaining buffers */
-		while (tx_desc != eop_desc) {
-			tx_buffer++;
-			tx_desc++;
-			i++;
-			if (unlikely(i == tx_ring->count)) {
-				i = 0;
-				tx_buffer = tx_ring->tx_buffer_info;
-				tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
-			}
+	struct libeth_sq_napi_stats stats = { };
 
-			/* unmap any remaining paged data */
-			if (dma_unmap_len(tx_buffer, len))
-				dma_unmap_page(tx_ring->dev,
-					       dma_unmap_addr(tx_buffer, dma),
-					       dma_unmap_len(tx_buffer, len),
-					       DMA_TO_DEVICE);
-		}
-
-		/* move us one more past the eop_desc for start of next pkt */
-		tx_buffer++;
-		i++;
-		if (unlikely(i == tx_ring->count)) {
-			i = 0;
-			tx_buffer = tx_ring->tx_buffer_info;
-		}
-	}
+	ixgbevf_clean_tx_num(tx_ring, false, ixgbevf_desc_used(tx_ring),
+			     &stats, NULL);
 
 	/* reset next_to_use and next_to_clean */
 	tx_ring->next_to_use = 0;
@@ -2035,10 +2086,17 @@ static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
 		ixgbevf_clean_xdp_ring(adapter->xdp_ring[i]);
 }
 
+void ixgbevf_flush_tx_queue(struct ixgbevf_ring *ring)
+{
+	u8 reg_idx = ring->reg_idx;
+
+	IXGBE_WRITE_REG(&ring->q_vector->adapter->hw, IXGBE_VFTXDCTL(reg_idx),
+			IXGBE_TXDCTL_SWFLSH);
+}
+
 void ixgbevf_down(struct ixgbevf_adapter *adapter)
 {
 	struct net_device *netdev = adapter->netdev;
-	struct ixgbe_hw *hw = &adapter->hw;
 	int i;
 
 	/* signal that we are down to the interrupt handler */
@@ -2064,19 +2122,11 @@ void ixgbevf_down(struct ixgbevf_adapter *adapter)
 	timer_delete_sync(&adapter->service_timer);
 
 	/* disable transmits in the hardware now that interrupts are off */
-	for (i = 0; i < adapter->num_tx_queues; i++) {
-		u8 reg_idx = adapter->tx_ring[i]->reg_idx;
-
-		IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx),
-				IXGBE_TXDCTL_SWFLSH);
-	}
-
-	for (i = 0; i < adapter->num_xdp_queues; i++) {
-		u8 reg_idx = adapter->xdp_ring[i]->reg_idx;
+	for (i = 0; i < adapter->num_tx_queues; i++)
+		ixgbevf_flush_tx_queue(adapter->tx_ring[i]);
 
-		IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx),
-				IXGBE_TXDCTL_SWFLSH);
-	}
+	for (i = 0; i < adapter->num_xdp_queues; i++)
+		ixgbevf_flush_tx_queue(adapter->xdp_ring[i]);
 
 	if (!pci_channel_offline(adapter->pdev))
 		ixgbevf_reset(adapter);
@@ -2153,6 +2203,56 @@ static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
 	return 0;
 }
 
+static void ixgbevf_cfg_queue_caps(struct ixgbevf_adapter *adapter)
+{
+	struct ixgbe_hw *hw = &adapter->hw;
+	u32 max_qpairs, rss;
+	int err;
+
+	adapter->q_caps = (typeof(adapter->q_caps)) {
+		.min_rxqs = 1,
+		.min_txqs = 1,
+		.max_rxqs = 1,
+		.max_txqs = 1,
+	};
+
+	/* fetch queue configuration from the PF */
+	spin_lock_bh(&adapter->mbx_lock);
+	err = ixgbevf_get_queues(hw, &adapter->q_caps.num_tcs,
+				 &adapter->q_caps.def_tc);
+	spin_unlock_bh(&adapter->mbx_lock);
+	if (err)
+		return;
+
+	/* we need as many queues as traffic classes */
+	if (adapter->q_caps.num_tcs > 1) {
+		adapter->q_caps.min_rxqs = adapter->q_caps.num_tcs;
+		adapter->q_caps.max_rxqs = adapter->q_caps.num_tcs;
+		return;
+	}
+
+	switch (hw->api_version) {
+	case ixgbe_mbox_api_11:
+	case ixgbe_mbox_api_12:
+	case ixgbe_mbox_api_13:
+	case ixgbe_mbox_api_14:
+	case ixgbe_mbox_api_15:
+	case ixgbe_mbox_api_16:
+	case ixgbe_mbox_api_17:
+		max_qpairs = min_t(u32, hw->mac.max_rx_queues,
+				        hw->mac.max_tx_queues);
+		break;
+	default:
+		max_qpairs = IXGBEVF_MAX_RSS_QUEUES;
+		break;
+	}
+
+	rss = min_t(u32, max_qpairs, num_online_cpus());
+
+	adapter->q_caps.max_rxqs = rss;
+	adapter->q_caps.max_txqs = rss;
+}
+
 /**
  * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
  * @adapter: board private structure to initialize
@@ -2166,51 +2266,29 @@ static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
  **/
 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
 {
-	struct ixgbe_hw *hw = &adapter->hw;
-	unsigned int def_q = 0;
-	unsigned int num_tcs = 0;
-	int err;
+	ixgbevf_cfg_queue_caps(adapter);
 
-	/* Start with base case */
-	adapter->num_rx_queues = 1;
-	adapter->num_tx_queues = 1;
-	adapter->num_xdp_queues = 0;
-
-	spin_lock_bh(&adapter->mbx_lock);
+	if (adapter->q_caps.num_tcs > 1) {
+		adapter->num_rx_queues = adapter->q_caps.max_rxqs;
+		adapter->num_tx_queues = adapter->q_caps.max_txqs;
+		adapter->num_xdp_queues = 0;
 
-	/* fetch queue configuration from the PF */
-	err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
-
-	spin_unlock_bh(&adapter->mbx_lock);
-
-	if (err)
 		return;
-
-	/* we need as many queues as traffic classes */
-	if (num_tcs > 1) {
-		adapter->num_rx_queues = num_tcs;
 	} else {
-		u16 rss = min_t(u16, num_online_cpus(), IXGBEVF_MAX_RSS_QUEUES);
-
-		switch (hw->api_version) {
-		case ixgbe_mbox_api_11:
-		case ixgbe_mbox_api_12:
-		case ixgbe_mbox_api_13:
-		case ixgbe_mbox_api_14:
-		case ixgbe_mbox_api_15:
-		case ixgbe_mbox_api_16:
-		case ixgbe_mbox_api_17:
-			if (adapter->xdp_prog &&
-			    hw->mac.max_tx_queues == rss)
-				rss = rss > 3 ? 2 : 1;
-
-			adapter->num_rx_queues = rss;
-			adapter->num_tx_queues = rss;
-			adapter->num_xdp_queues = adapter->xdp_prog ? rss : 0;
-			break;
-		default:
-			break;
-		}
+		u32 max_qpairs = min_t(u32, adapter->q_caps.max_rxqs,
+					    adapter->q_caps.max_txqs);
+
+		adapter->num_req_qpairs =
+			adapter->num_req_qpairs ? : adapter->q_caps.max_rxqs;
+		WARN_ON_ONCE(adapter->num_req_qpairs > max_qpairs);
+		adapter->num_rx_queues =
+			min_t(u32, adapter->num_req_qpairs, max_qpairs);
+		adapter->num_tx_queues = adapter->num_rx_queues;
+		// if (adapter->num_rx_queues + adapter->num_tx_queues <=
+		//     adapter->q_caps.max_txqs && adapter->xdp_prog)
+		// 	adapter->num_xdp_queues = adapter->num_rx_queues;
+		// else
+			adapter->num_xdp_queues = 0;
 	}
 }
 
@@ -2522,7 +2600,7 @@ static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
  * @adapter: board private structure to initialize
  *
  **/
-static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
+int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
 {
 	int err;
 
@@ -2563,7 +2641,7 @@ static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
  * We go through and clear interrupt specific resources and reset the structure
  * to pre-load conditions
  **/
-static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
+void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
 {
 	adapter->num_tx_queues = 0;
 	adapter->num_xdp_queues = 0;
@@ -2929,8 +3007,8 @@ void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring)
 	else
 		ixgbevf_clean_xdp_ring(tx_ring);
 
-	vfree(tx_ring->tx_buffer_info);
-	tx_ring->tx_buffer_info = NULL;
+	vfree(tx_ring->tx_sqes);
+	tx_ring->tx_sqes = NULL;
 
 	/* if not set, then don't free */
 	if (!tx_ring->desc)
@@ -2971,11 +3049,10 @@ int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring)
 	struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev);
 	int size;
 
-	size = (!ring_is_xdp(tx_ring) ? sizeof(struct ixgbevf_tx_buffer) :
-		sizeof(struct libeth_sqe)) * tx_ring->count;
+	size = sizeof(struct libeth_sqe) * tx_ring->count;
 
-	tx_ring->tx_buffer_info = vmalloc(size);
-	if (!tx_ring->tx_buffer_info)
+	tx_ring->tx_sqes = vmalloc(size);
+	if (!tx_ring->tx_sqes)
 		goto err;
 
 	u64_stats_init(&tx_ring->syncp);
@@ -2992,8 +3069,8 @@ int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring)
 	return 0;
 
 err:
-	vfree(tx_ring->tx_buffer_info);
-	tx_ring->tx_buffer_info = NULL;
+	vfree(tx_ring->tx_sqes);
+	tx_ring->tx_sqes = NULL;
 	hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit descriptor ring\n");
 	return -ENOMEM;
 }
@@ -3313,11 +3390,10 @@ static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
 	context_desc->mss_l4len_idx	= cpu_to_le32(mss_l4len_idx);
 }
 
-static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
-		       struct ixgbevf_tx_buffer *first,
-		       u8 *hdr_len,
-		       struct ixgbevf_ipsec_tx_data *itd)
+static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, struct libeth_sqe *first,
+		       u8 *hdr_len, struct ixgbevf_ipsec_tx_data *itd)
 {
+	struct ixgbevf_skb_sqe_priv *priv = (void *)&first->priv;
 	u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
 	struct sk_buff *skb = first->skb;
 	union {
@@ -3343,7 +3419,7 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
 	if (err < 0)
 		return err;
 
-	if (eth_p_mpls(first->protocol))
+	if (eth_p_mpls(priv->protocol))
 		ip.hdr = skb_inner_network_header(skb);
 	else
 		ip.hdr = skb_network_header(skb);
@@ -3368,12 +3444,12 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
 		type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
 
 		ip.v4->tot_len = 0;
-		first->tx_flags |= IXGBE_TX_FLAGS_TSO |
+		priv->tx_flags |= IXGBE_TX_FLAGS_TSO |
 				   IXGBE_TX_FLAGS_CSUM |
 				   IXGBE_TX_FLAGS_IPV4;
 	} else {
 		ip.v6->payload_len = 0;
-		first->tx_flags |= IXGBE_TX_FLAGS_TSO |
+		priv->tx_flags |= IXGBE_TX_FLAGS_TSO |
 				   IXGBE_TX_FLAGS_CSUM;
 	}
 
@@ -3388,8 +3464,8 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
 	csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen));
 
 	/* update gso size and bytecount with header size */
-	first->gso_segs = skb_shinfo(skb)->gso_segs;
-	first->bytecount += (first->gso_segs - 1) * *hdr_len;
+	first->packets = skb_shinfo(skb)->gso_segs;
+	first->bytes += (first->packets - 1) * *hdr_len;
 
 	/* mss_l4len_id: use 1 as index for TSO */
 	mss_l4len_idx = (*hdr_len - l4_offset) << IXGBE_ADVTXD_L4LEN_SHIFT;
@@ -3402,7 +3478,7 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
 	/* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
 	vlan_macip_lens = l4.hdr - ip.hdr;
 	vlan_macip_lens |= (ip.hdr - skb->data) << IXGBE_ADVTXD_MACLEN_SHIFT;
-	vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
+	vlan_macip_lens |= priv->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
 
 	ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, fceof_saidx, type_tucmd,
 			    mss_l4len_idx);
@@ -3411,9 +3487,10 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
 }
 
 static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
-			    struct ixgbevf_tx_buffer *first,
+			    struct libeth_sqe *first,
 			    struct ixgbevf_ipsec_tx_data *itd)
 {
+	struct ixgbevf_skb_sqe_priv *priv = (void *)&first->priv;
 	struct sk_buff *skb = first->skb;
 	u32 vlan_macip_lens = 0;
 	u32 fceof_saidx = 0;
@@ -3440,17 +3517,17 @@ static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
 		goto no_csum;
 	}
 
-	if (first->protocol == htons(ETH_P_IP))
+	if (priv->protocol == htons(ETH_P_IP))
 		type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
 
 	/* update TX checksum flag */
-	first->tx_flags |= IXGBE_TX_FLAGS_CSUM;
+	priv->tx_flags |= IXGBE_TX_FLAGS_CSUM;
 	vlan_macip_lens = skb_checksum_start_offset(skb) -
 			  skb_network_offset(skb);
 no_csum:
 	/* vlan_macip_lens: MACLEN, VLAN tag */
 	vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
-	vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
+	vlan_macip_lens |= priv->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
 
 	fceof_saidx |= itd->pfsa;
 	type_tucmd |= itd->flags | itd->trailer_len;
@@ -3507,20 +3584,21 @@ static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
 }
 
 static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
-			   struct ixgbevf_tx_buffer *first,
+			   struct libeth_sqe *first,
 			   const u8 hdr_len)
 {
+	struct ixgbevf_skb_sqe_priv *priv = (void *)&first->priv;
 	struct sk_buff *skb = first->skb;
-	struct ixgbevf_tx_buffer *tx_buffer;
 	union ixgbe_adv_tx_desc *tx_desc;
+	u32 tx_flags = priv->tx_flags;
+	struct libeth_sqe *used_sqe;
 	skb_frag_t *frag;
 	dma_addr_t dma;
 	unsigned int data_len, size;
-	u32 tx_flags = first->tx_flags;
 	__le32 cmd_type = ixgbevf_tx_cmd_type(tx_flags);
-	u16 i = tx_ring->next_to_use;
+	u16 ntu = tx_ring->next_to_use;
 
-	tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
+	tx_desc = IXGBEVF_TX_DESC(tx_ring, ntu);
 
 	ixgbevf_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len);
 
@@ -3529,15 +3607,16 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
 
 	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
 
-	tx_buffer = first;
-
 	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
-		if (dma_mapping_error(tx_ring->dev, dma))
+
+		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
 			goto dma_error;
 
 		/* record length, and DMA address */
-		dma_unmap_len_set(tx_buffer, len, size);
-		dma_unmap_addr_set(tx_buffer, dma, dma);
+		used_sqe = &tx_ring->tx_sqes[ntu];
+		dma_unmap_len_set(used_sqe, len, size);
+		dma_unmap_addr_set(used_sqe, dma, dma);
+		used_sqe->type = LIBETH_SQE_FRAG;
 
 		tx_desc->read.buffer_addr = cpu_to_le64(dma);
 
@@ -3545,11 +3624,11 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
 			tx_desc->read.cmd_type_len =
 				cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD);
 
-			i++;
+			ntu++;
 			tx_desc++;
-			if (i == tx_ring->count) {
+			if (ntu == tx_ring->count) {
 				tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
-				i = 0;
+				ntu = 0;
 			}
 			tx_desc->read.olinfo_status = 0;
 
@@ -3564,11 +3643,11 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
 
 		tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
 
-		i++;
+		ntu++;
 		tx_desc++;
-		if (i == tx_ring->count) {
+		if (ntu == tx_ring->count) {
 			tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
-			i = 0;
+			ntu = 0;
 		}
 		tx_desc->read.olinfo_status = 0;
 
@@ -3577,18 +3656,14 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
 
 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
 				       DMA_TO_DEVICE);
-
-		tx_buffer = &tx_ring->tx_buffer_info[i];
 	}
 
 	/* write last descriptor with RS and EOP bits */
 	cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD);
 	tx_desc->read.cmd_type_len = cmd_type;
 
-	/* set the timestamp */
-	first->time_stamp = jiffies;
-
 	skb_tx_timestamp(skb);
+	first->type = LIBETH_SQE_SKB;
 
 	/* Force memory writes to complete before letting h/w know there
 	 * are new descriptors to fetch.  (Only applicable for weak-ordered
@@ -3600,47 +3675,48 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
 	wmb();
 
 	/* set next_to_watch value indicating a packet is present */
-	first->next_to_watch = tx_desc;
+	tx_ring->xdp_sqes[tx_ring->cached_ntu].rs_idx = ntu + 1;
 
-	i++;
-	if (i == tx_ring->count)
-		i = 0;
+	ntu++;
+	if (ntu == tx_ring->count)
+		ntu = 0;
 
-	tx_ring->next_to_use = i;
+	tx_ring->next_to_use = ntu;
+	tx_ring->cached_ntu = ntu;
 
 	/* notify HW of packet */
-	ixgbevf_write_tail(tx_ring, i);
+	ixgbevf_write_tail(tx_ring, ntu);
 
 	return;
 dma_error:
 	dev_err(tx_ring->dev, "TX DMA map failed\n");
-	tx_buffer = &tx_ring->tx_buffer_info[i];
+	used_sqe = &tx_ring->tx_sqes[ntu];
 
 	/* clear dma mappings for failed tx_buffer_info map */
-	while (tx_buffer != first) {
-		if (dma_unmap_len(tx_buffer, len))
+	while (used_sqe != first) {
+		if (dma_unmap_len(used_sqe, len))
 			dma_unmap_page(tx_ring->dev,
-				       dma_unmap_addr(tx_buffer, dma),
-				       dma_unmap_len(tx_buffer, len),
+				       dma_unmap_addr(used_sqe, dma),
+				       dma_unmap_len(used_sqe, len),
 				       DMA_TO_DEVICE);
-		dma_unmap_len_set(tx_buffer, len, 0);
+		dma_unmap_len_set(used_sqe, len, 0);
 
-		if (i-- == 0)
-			i += tx_ring->count;
-		tx_buffer = &tx_ring->tx_buffer_info[i];
+		if (ntu-- == 0)
+			ntu += tx_ring->count;
+		used_sqe = &tx_ring->tx_sqes[ntu];
 	}
 
-	if (dma_unmap_len(tx_buffer, len))
+	if (dma_unmap_len(used_sqe, len))
 		dma_unmap_single(tx_ring->dev,
-				 dma_unmap_addr(tx_buffer, dma),
-				 dma_unmap_len(tx_buffer, len),
+				 dma_unmap_addr(used_sqe, dma),
+				 dma_unmap_len(used_sqe, len),
 				 DMA_TO_DEVICE);
-	dma_unmap_len_set(tx_buffer, len, 0);
+	dma_unmap_len_set(used_sqe, len, 0);
 
-	dev_kfree_skb_any(tx_buffer->skb);
-	tx_buffer->skb = NULL;
+	dev_kfree_skb_any(used_sqe->skb);
+	used_sqe->skb = NULL;
 
-	tx_ring->next_to_use = i;
+	tx_ring->next_to_use = ntu;
 }
 
 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
@@ -3675,7 +3751,8 @@ static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
 static int ixgbevf_xmit_frame_ring(struct sk_buff *skb,
 				   struct ixgbevf_ring *tx_ring)
 {
-	struct ixgbevf_tx_buffer *first;
+	struct ixgbevf_skb_sqe_priv *priv;
+	struct libeth_sqe *first;
 	int tso;
 	u32 tx_flags = 0;
 	u16 count = TXD_USE_COUNT(skb_headlen(skb));
@@ -3712,10 +3789,11 @@ static int ixgbevf_xmit_frame_ring(struct sk_buff *skb,
 	}
 
 	/* record the location of the first descriptor for this packet */
-	first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
+	first = &tx_ring->tx_sqes[tx_ring->next_to_use];
+	priv = (void *)&first->priv;
 	first->skb = skb;
-	first->bytecount = skb->len;
-	first->gso_segs = 1;
+	first->bytes = skb->len;
+	first->packets = 1;
 
 	if (skb_vlan_tag_present(skb)) {
 		tx_flags |= skb_vlan_tag_get(skb);
@@ -3724,8 +3802,8 @@ static int ixgbevf_xmit_frame_ring(struct sk_buff *skb,
 	}
 
 	/* record initial flags and protocol */
-	first->tx_flags = tx_flags;
-	first->protocol = vlan_get_protocol(skb);
+	priv->tx_flags = tx_flags;
+	priv->protocol = vlan_get_protocol(skb);
 
 #ifdef CONFIG_IXGBEVF_IPSEC
 	if (xfrm_offload(skb) && !ixgbevf_ipsec_tx(tx_ring, first, &ipsec_tx))
@@ -4047,6 +4125,9 @@ static int ixgbevf_xdp(struct net_device *dev, struct netdev_bpf *xdp)
 	switch (xdp->command) {
 	case XDP_SETUP_PROG:
 		return ixgbevf_xdp_setup(dev, xdp->prog, xdp->extack);
+	case XDP_SETUP_XSK_POOL:
+		return ixgbevf_setup_xsk_pool(netdev_priv(dev), xdp->xsk.pool,
+					      xdp->xsk.queue_id);
 	default:
 		return -EINVAL;
 	}
@@ -4067,6 +4148,7 @@ static const struct net_device_ops ixgbevf_netdev_ops = {
 	.ndo_features_check	= ixgbevf_features_check,
 	.ndo_bpf		= ixgbevf_xdp,
 	.ndo_xdp_xmit		= ixgbevf_xdp_xmit,
+	.ndo_xsk_wakeup		= ixgbevf_xsk_wakeup,
 };
 
 static void ixgbevf_assign_netdev_ops(struct net_device *dev)
@@ -4199,7 +4281,7 @@ static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 			    NETIF_F_HW_VLAN_CTAG_TX;
 
 	netdev->priv_flags |= IFF_UNICAST_FLT;
-	libeth_xdp_set_features_noredir(netdev, NULL, 0, NULL);
+	libeth_xdp_set_features_noredir(netdev, NULL, IXGBEVF_XSK_MAX_ZC_FRAGS);
 
 	/* MTU range: 68 - 1504 or 9710 */
 	netdev->min_mtu = ETH_MIN_MTU;
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h
index 851d1c054d9810..ac36fac9f79717 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h
@@ -24,6 +24,7 @@ static inline bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring,
 	/* fetch, update, and store next to clean */
 	ntc = (ntc < rx_ring->count) ? ntc : 0;
 	rx_ring->next_to_clean = ntc;
+	rx_ring->pending++;
 
 	prefetch(IXGBEVF_RX_DESC(rx_ring, ntc));
 
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h
index da4f397944ed88..452b0013898918 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h
@@ -4,33 +4,36 @@
 #ifndef _IXGBEVF_XDP_LIB_H_
 #define _IXGBEVF_XDP_LIB_H_
 
-#include <net/libeth/xdp.h>
+#include <net/libeth/xsk.h>
 
 #include "ixgbevf.h"
 
-static inline u16 ixgbevf_tx_get_num_sent(struct ixgbevf_ring *xdp_ring)
+static inline u16 ixgbevf_tx_get_num_sent(struct ixgbevf_ring *tx_ring,
+					  u16 budget)
 {
-	u16 ntc = xdp_ring->next_to_clean;
+	u16 ntc = tx_ring->next_to_clean;
 	u16 to_clean = 0;
 
-	while (likely(to_clean < xdp_ring->pending)) {
-		u32 idx = xdp_ring->xdp_sqes[ntc].rs_idx;
+	while (likely(to_clean < budget)) {
+		u32 idx = tx_ring->xdp_sqes[ntc].rs_idx;
 		union ixgbe_adv_tx_desc *rs_desc;
 
 		if (!idx--)
 			break;
 
-		rs_desc = IXGBEVF_TX_DESC(xdp_ring, idx);
+		smp_rmb();
+
+		rs_desc = IXGBEVF_TX_DESC(tx_ring, idx);
 
 		if (!(rs_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
 			break;
 
-		xdp_ring->xdp_sqes[ntc].rs_idx = 0;
+		tx_ring->xdp_sqes[ntc].rs_idx = 0;
 
 		to_clean +=
-			(idx >= ntc ? idx : idx + xdp_ring->count) - ntc + 1;
+			(idx >= ntc ? idx : idx + tx_ring->count) - ntc + 1;
 
-		ntc = (idx + 1 == xdp_ring->count) ? 0 : idx + 1;
+		ntc = (idx + 1 == tx_ring->count) ? 0 : idx + 1;
 	}
 
 	return to_clean;
@@ -40,6 +43,7 @@ static inline void ixgbevf_clean_xdp_num(struct ixgbevf_ring *xdp_ring,
 					 bool in_napi, u16 to_clean)
 {
 	struct libeth_xdpsq_napi_stats stats = { };
+	bool xsk_ring = ring_is_xsk(xdp_ring);
 	u32 ntc = xdp_ring->next_to_clean;
 	struct xdp_frame_bulk cbulk;
 	struct libeth_cq_pp cp = {
@@ -48,11 +52,14 @@ static inline void ixgbevf_clean_xdp_num(struct ixgbevf_ring *xdp_ring,
 		.xss = &stats,
 		.napi = in_napi,
 	};
+	u32 xsk_frames = 0;
 
 	xdp_frame_bulk_init(&cbulk);
 	xdp_ring->pending -= to_clean;
 
 	while (likely(to_clean--)) {
+		xsk_frames += xsk_ring &&
+			likely(!xdp_ring->xdp_sqes[ntc].type) ? 1 : 0;
 		libeth_xdp_complete_tx(&xdp_ring->xdp_sqes[ntc], &cp);
 		ntc++;
 		ntc = unlikely(ntc == xdp_ring->count) ? 0 : ntc;
@@ -60,6 +67,8 @@ static inline void ixgbevf_clean_xdp_num(struct ixgbevf_ring *xdp_ring,
 
 	xdp_ring->next_to_clean = ntc;
 	xdp_flush_frame_bulk(&cbulk);
+	if (xsk_frames)
+		xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames);
 }
 
 static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq)
@@ -67,8 +76,9 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq)
 	struct ixgbevf_ring *xdp_ring = xdpsq;
 
 	libeth_xdpsq_lock(&xdp_ring->xdpq_lock);
-	if (unlikely(ixgbevf_desc_unused(xdp_ring) < LIBETH_XDP_TX_BULK)) {
-		u16 to_clean = ixgbevf_tx_get_num_sent(xdp_ring);
+	if (unlikely(ixgbevf_desc_unused(xdp_ring) < xdp_ring->thresh)) {
+		u16 to_clean =
+			ixgbevf_tx_get_num_sent(xdp_ring, xdp_ring->pending);
 
 		if (likely(to_clean))
 			ixgbevf_clean_xdp_num(xdp_ring, true, to_clean);
@@ -91,6 +101,7 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq)
 
 		xdp_ring->next_to_use = 1;
 		xdp_ring->pending = 1;
+		xdp_ring->xdp_sqes[0].type = LIBETH_SQE_CTX;
 
 		/* Finish descriptor writes before bumping tail */
 		wmb();
@@ -103,38 +114,134 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq)
 		.lock = &xdp_ring->xdpq_lock,
 		.ntu = &xdp_ring->next_to_use,
 		.pending = &xdp_ring->pending,
-		.pool = NULL,
+		.pool = xdp_ring->xsk_pool,
 		.sqes = xdp_ring->xdp_sqes,
 	};
 
 	return ixgbevf_desc_unused(xdp_ring);
 }
 
+static inline struct netdev_queue *
+ixgbevf_xdp_tx_get_nq(struct ixgbevf_ring *tx_ring)
+{
+	struct netdev_queue *nq;
+
+	nq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->queue_index);
+	__netif_tx_lock(nq, smp_processor_id());
+
+	return nq;
+}
+
+static inline void ixgbevf_xdp_tx_put_nq(struct ixgbevf_ring *tx_ring)
+{
+	struct netdev_queue *nq;
+
+	nq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->queue_index);
+	__netif_tx_unlock(nq);
+}
+
+static inline u32 ixgbevf_prep_tx_sq(void *xdpsq, struct libeth_xdpsq *sq)
+{
+	struct ixgbe_adv_tx_context_desc *context_desc;
+	struct ixgbevf_ring *tx_ring = xdpsq;
+	struct netdev_queue *nq;
+	u32 num_unused, ntu;
+
+	/* Serialization of producers in .ndo_start_xmit(),
+	 * .ndo_xdp_xmit() and XDP_TX
+	 */
+	nq = ixgbevf_xdp_tx_get_nq(tx_ring);
+
+	/* We need at least 1 additional descriptor for context */
+	num_unused = ixgbevf_desc_unused(tx_ring);
+	if (num_unused < 2)
+		return 0;
+
+	/* Inform the stack that queue is transmitting to avoid Tx timeout */
+	txq_trans_cond_update(nq);
+
+	/* Shared TxQ cleaning was done beforehand */
+
+	/* Instead of sending a context descriptor once for an XDP-only ring,
+	 * do this before sending each bulk
+	 */
+	ntu = tx_ring->next_to_use;
+	context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, ntu);
+	tx_ring->xdp_sqes[ntu].type = LIBETH_SQE_CTX;
+	context_desc->vlan_macip_lens =
+		cpu_to_le32(ETH_HLEN << IXGBE_ADVTXD_MACLEN_SHIFT);
+	context_desc->fceof_saidx = 0;
+	context_desc->type_tucmd_mlhl =
+		cpu_to_le32(IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT);
+	context_desc->mss_l4len_idx = 0;
+
+	ntu++;
+	num_unused--;
+	ntu = ntu == tx_ring->count ? 0 : ntu;
+	tx_ring->next_to_use = ntu;
+
+	*sq = (struct libeth_xdpsq) {
+		.count = tx_ring->count,
+		.descs = tx_ring->desc,
+		.lock = &tx_ring->xdpq_lock,
+		.ntu = &tx_ring->next_to_use,
+		/* This value is ignored in shared queues */
+		.pending = &tx_ring->pending,
+		.pool = tx_ring->xsk_pool,
+		.sqes = tx_ring->xdp_sqes,
+	};
+
+	return num_unused;
+}
+
+static inline void
+ixgbevf_xdp_tx_unprep(void *xdpsq, struct libeth_xdpsq *sq __always_unused)
+{
+	ixgbevf_xdp_tx_put_nq(xdpsq);
+}
+
 static inline void ixgbevf_xdp_rs_and_bump(void *xdpsq, bool sent, bool flush)
 {
 	struct ixgbevf_ring *xdp_ring = xdpsq;
 	union ixgbe_adv_tx_desc *desc;
+	bool is_shared;
 	u32 ltu;
 
-	libeth_xdpsq_lock(&xdp_ring->xdpq_lock);
+	is_shared = !test_bit(__IXGBEVF_TX_XDP_RING, &xdp_ring->state);
+
+	if (is_shared)
+		ixgbevf_xdp_tx_get_nq(xdp_ring);
+	else
+		libeth_xdpsq_lock(&xdp_ring->xdpq_lock);
 
 	if ((!flush && xdp_ring->pending < xdp_ring->count - 1) ||
 	    xdp_ring->cached_ntu == xdp_ring->next_to_use)
 		goto unlock;
 
 	ltu = (xdp_ring->next_to_use ? : xdp_ring->count) - 1;
+
+	/* We will not get DD on a context descriptor */
+	if (unlikely(xdp_ring->xdp_sqes[ltu].type == LIBETH_SQE_CTX))
+		goto unlock;
+
 	desc = IXGBEVF_TX_DESC(xdp_ring, ltu);
 	desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD);
 
 	xdp_ring->xdp_sqes[xdp_ring->cached_ntu].rs_idx = ltu + 1;
 	xdp_ring->cached_ntu = xdp_ring->next_to_use;
 
+	/* In case the packet was interrupted, discard it */
+	xdp_ring->xdp_sqes[ltu].priv = 0;
+
 	/* Finish descriptor writes before bumping tail */
 	wmb();
 	ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use);
 
 unlock:
-	libeth_xdpsq_unlock(&xdp_ring->xdpq_lock);
+	if (is_shared)
+		ixgbevf_xdp_tx_put_nq(xdp_ring);
+	else
+		libeth_xdpsq_unlock(&xdp_ring->xdpq_lock);
 }
 
 #endif /* _IXGBEVF_XDP_LIB_H_ */
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c
new file mode 100644
index 00000000000000..f3edeae6aa00a6
--- /dev/null
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c
@@ -0,0 +1,383 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (C) 2026 Intel Corporation */
+
+#include <net/libeth/xsk.h>
+
+#include "ixgbevf_txrx_lib.h"
+#include "ixgbevf_xdp_lib.h"
+#include "ixgbevf_xsk.h"
+
+/**
+ * ixgbevf_single_irq_disable - Mask off interrupt generation on a single vector
+ * @adapter: board private structure
+ * @vidx: vector id
+ **/
+static void ixgbevf_single_irq_disable(struct ixgbevf_adapter *adapter,
+				       u16 vidx)
+{
+	struct ixgbe_hw *hw = &adapter->hw;
+
+	IXGBE_WRITE_REG(hw, IXGBE_VTEIAM,
+			adapter->eims_enable_mask & ~BIT(vidx));
+	IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, BIT(vidx));
+	IXGBE_WRITE_REG(hw, IXGBE_VTEIAC,
+			adapter->eims_enable_mask & ~BIT(vidx));
+
+	IXGBE_WRITE_FLUSH(hw);
+
+	synchronize_irq(adapter->msix_entries[vidx].vector);
+}
+
+static void ixgbevf_qp_dis(struct ixgbevf_adapter *adapter, u16 qid)
+{
+	struct ixgbevf_ring *tx_ring, *rx_ring = adapter->rx_ring[qid];
+	struct ixgbevf_q_vector *q_vector = rx_ring->q_vector;
+
+	netif_stop_subqueue(adapter->netdev, qid);
+	ixgbevf_single_irq_disable(adapter, q_vector->v_idx);
+	napi_disable(&q_vector->napi);
+
+	synchronize_net();
+
+	ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[qid]);
+	ixgbevf_clean_rx_ring(rx_ring);
+	ixgbevf_rx_destroy_pp(rx_ring);
+
+	/* Clean both XDP and normal Tx queue */
+	ixgbevf_for_each_ring(tx_ring, q_vector->tx) {
+		ixgbevf_flush_tx_queue(tx_ring);
+		if (ring_is_xdp(tx_ring))
+			ixgbevf_clean_xdp_ring(tx_ring);
+		else
+			ixgbevf_clean_tx_ring(tx_ring);
+	}
+}
+
+static void ixgbevf_qp_ena(struct ixgbevf_adapter *adapter, u16 qid)
+{
+	struct ixgbevf_ring *tx_ring, *rx_ring = adapter->rx_ring[qid];
+	struct ixgbevf_q_vector *q_vector = rx_ring->q_vector;
+
+	ixgbevf_configure_rx_ring(adapter, rx_ring);
+	ixgbevf_for_each_ring(tx_ring, q_vector->tx)
+		ixgbevf_configure_tx_ring(adapter, tx_ring);
+
+	napi_enable(&q_vector->napi);
+	ixgbevf_irq_enable(adapter);
+	netif_start_subqueue(adapter->netdev, qid);
+	napi_schedule(&q_vector->napi);
+}
+
+int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter,
+			   struct xsk_buff_pool *pool, u16 qid)
+{
+	bool running = !test_bit(__IXGBEVF_DOWN, &adapter->state) &&
+		       adapter->xdp_prog;
+	int err;
+
+	if (running)
+		ixgbevf_qp_dis(adapter, qid);
+
+	err = libeth_xsk_setup_pool(adapter->netdev, qid, !!pool);
+
+	if (running)
+		ixgbevf_qp_ena(adapter, qid);
+
+	return err;
+}
+
+static void ixgbevf_fill_rx_xsk_desc(const struct libeth_xskfq_fp *fq, u32 i)
+{
+	union ixgbe_adv_rx_desc *rx_desc =
+		&((union ixgbe_adv_rx_desc *)fq->descs)[i];
+
+	rx_desc->read.pkt_addr =
+		cpu_to_le64(libeth_xsk_buff_xdp_get_dma(fq->fqes[i]));
+	rx_desc->wb.upper.length = 0;
+}
+
+bool ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num)
+{
+	struct libeth_xskfq_fp fq = {
+		.count = rx_ring->count,
+		.descs = rx_ring->desc,
+		.fqes = rx_ring->xsk_fqes,
+		.ntu = rx_ring->next_to_use,
+		.pool = rx_ring->xsk_pool,
+	};
+	u32 done;
+
+	done = libeth_xskfqe_alloc(&fq, num, ixgbevf_fill_rx_xsk_desc);
+	if (likely(done)) {
+		/* Finish descriptor writes before bumping tail */
+		wmb();
+		ixgbevf_write_tail(rx_ring, fq.ntu);
+	}
+
+	rx_ring->next_to_use = fq.ntu;
+	rx_ring->pending -= done;
+
+	return done == num;
+}
+
+void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring)
+{
+	u32 ntc = rx_ring->next_to_clean;
+
+	if (rx_ring->xsk_xdp_head)
+		xsk_buff_free(&rx_ring->xsk_xdp_head->base);
+
+	rx_ring->xsk_xdp_head = NULL;
+
+	while (ntc != rx_ring->next_to_use) {
+		xsk_buff_free(&rx_ring->xsk_fqes[ntc]->base);
+		ntc++;
+		ntc = ntc == rx_ring->count ? 0 : ntc;
+	}
+}
+
+struct ixgbevf_zc_sqe_priv {
+	u16 first_desc;
+	u16 len;
+};
+
+static_assert(sizeof(struct ixgbevf_zc_sqe_priv) <=
+	      sizeof_field(struct libeth_sqe, priv));
+
+static void ixgbevf_xsk_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i,
+				  const struct libeth_xdpsq *sq, u64 priv)
+{
+	union ixgbe_adv_tx_desc *descs = sq->descs, *tx_desc = &descs[i];
+	u32 ltu = (i ? : sq->count) - 1;
+
+	u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA |
+		       IXGBE_ADVTXD_DCMD_DEXT |
+		       IXGBE_ADVTXD_DCMD_IFCS |
+		       desc.len;
+
+	tx_desc->read.buffer_addr = cpu_to_le64(desc.addr);
+
+	if (likely((desc.flags & LIBETH_XDP_TX_LAST) && !sq->sqes[ltu].priv)) {
+		tx_desc->read.olinfo_status =
+			cpu_to_le32((desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT) |
+				    IXGBE_ADVTXD_CC);
+		tx_desc->read.cmd_type_len =
+			cpu_to_le32(cmd_type | IXGBE_TXD_CMD_EOP);
+		return;
+	}
+
+	/* No previous packet */
+	if (!sq->sqes[ltu].priv) {
+		struct ixgbevf_zc_sqe_priv *sqe_priv =
+						(void *)&sq->sqes[i].priv;
+
+		sqe_priv->first_desc = i;
+		sqe_priv->len = desc.len;
+
+		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
+
+		return;
+	}
+
+	if (sq->sqes[ltu].priv) {
+		struct ixgbevf_zc_sqe_priv *sqe_priv =
+						(void *)&sq->sqes[i].priv;
+
+		sq->sqes[i].priv = sq->sqes[ltu].priv;
+		sq->sqes[ltu].priv = 0;
+		sqe_priv->len += desc.len;
+
+		if (desc.flags & LIBETH_XDP_TX_LAST) {
+			union ixgbe_adv_tx_desc *first_desc =
+						&descs[sqe_priv->first_desc];
+
+			first_desc->read.olinfo_status =
+				cpu_to_le32((sqe_priv->len <<
+					     IXGBE_ADVTXD_PAYLEN_SHIFT) |
+					    IXGBE_ADVTXD_CC);
+			tx_desc->read.cmd_type_len =
+				cpu_to_le32(cmd_type | IXGBE_TXD_CMD_EOP);
+			cmd_type |= IXGBE_TXD_CMD_EOP;
+			sq->sqes[i].priv = 0;
+		}
+
+		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
+	}
+}
+
+LIBETH_XDP_DEFINE_START();
+LIBETH_XSK_DEFINE_FLUSH_TX(static ixgbevf_xsk_flush_tx, ixgbevf_prep_xdp_sq,
+			   ixgbevf_xsk_xmit_desc);
+__LIBETH_XDP_DEFINE_FLUSH_TX(static ixgbevf_xsk_flush_tx_shared,
+			     ixgbevf_prep_tx_sq, ixgbevf_xsk_xmit_desc,
+			     libeth_xdp_tx_unprep, xsk);
+LIBETH_XDP_DEFINE_END();
+
+static bool ixgbevf_xsk_flush_common(struct libeth_xdp_tx_bulk *bq, u32 flags)
+{
+	const struct ixgbevf_ring *ring = bq->xdpsq;
+
+	if (test_bit(__IXGBEVF_TX_XDP_RING, &ring->state))
+		return ixgbevf_xsk_flush_tx(bq, flags);
+	else
+		return ixgbevf_xsk_flush_tx_shared(bq, flags);
+}
+
+LIBETH_XDP_DEFINE_START();
+LIBETH_XSK_DEFINE_RUN_PROG(static ixgbevf_xsk_run_prog,
+			   ixgbevf_xsk_flush_common);
+LIBETH_XSK_DEFINE_FINALIZE(static ixgbevf_xsk_finalize_xdp_napi,
+			   ixgbevf_xsk_flush_common, ixgbevf_xdp_rs_and_bump);
+LIBETH_XDP_DEFINE_END();
+
+u32 ixgbevf_clean_xsk_rx_irq(struct ixgbevf_q_vector *q_vector,
+			     struct ixgbevf_ring *rx_ring, int budget)
+{
+	struct ixgbevf_adapter *adapter = q_vector->adapter;
+	int num_xdp_queues = adapter->num_xdp_queues;
+	u32 total_rx_bytes = 0, total_rx_packets = 0;
+	LIBETH_XDP_ONSTACK_BULK(xdp_tx_bulk);
+	struct libeth_xdp_buff *head_xdp;
+	bool failure = false, wake;
+	struct sk_buff *skb;
+
+	wake = xsk_uses_need_wakeup(rx_ring->xsk_pool);
+	if (wake)
+		xsk_clear_rx_need_wakeup(rx_ring->xsk_pool);
+
+	if (num_xdp_queues)
+		libeth_xsk_tx_init_bulk(&xdp_tx_bulk, rx_ring->xdp_prog,
+					adapter->netdev, adapter->xdp_ring,
+					num_xdp_queues);
+	else
+		libeth_xsk_tx_init_bulk_shared(&xdp_tx_bulk, rx_ring->xdp_prog,
+					       adapter->netdev,
+					       adapter->tx_ring,
+					       adapter->num_tx_queues);
+
+	while (likely(total_rx_packets < budget)) {
+		union ixgbe_adv_rx_desc *rx_desc;
+		struct libeth_xdp_buff *rx_buffer;
+		unsigned int size;
+		u32 xdp_result;
+
+		rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean);
+		size = le16_to_cpu(rx_desc->wb.upper.length);
+		if (unlikely(!size))
+			break;
+
+		/* Avoid reading other descriptor fields before checking size */
+		rmb();
+
+		rx_buffer = rx_ring->xsk_fqes[rx_ring->next_to_clean];
+		head_xdp = libeth_xsk_process_buff(head_xdp, rx_buffer, size);
+		if (unlikely(!head_xdp) || ixgbevf_is_non_eop(rx_ring, rx_desc))
+			continue;
+
+		total_rx_packets++;
+		total_rx_bytes += xdp_get_buff_len(&head_xdp->base);
+
+		xdp_result = ixgbevf_xsk_run_prog(head_xdp, &xdp_tx_bulk);
+		if (xdp_result) {
+			head_xdp = NULL;
+			if (likely(xdp_result != LIBETH_XDP_ABORTED))
+				continue;
+			failure = true;
+			break;
+		}
+
+		skb = xdp_build_skb_from_zc(&head_xdp->base);
+
+		if (unlikely(!skb)) {
+			libeth_xdp_return_buff_slow(head_xdp);
+			head_xdp = NULL;
+			rx_ring->rx_stats.alloc_rx_buff_failed++;
+			break;
+		}
+
+		head_xdp = NULL;
+
+		if (unlikely(ixgbevf_cleanup_headers(rx_ring, rx_desc, skb))) {
+			skb = NULL;
+			continue;
+		}
+
+		if (unlikely((skb->pkt_type == PACKET_BROADCAST ||
+			      skb->pkt_type == PACKET_MULTICAST) &&
+			     ether_addr_equal(rx_ring->netdev->dev_addr,
+					      eth_hdr(skb)->h_source))) {
+			dev_kfree_skb_irq(skb);
+			continue;
+		}
+
+		/* populate checksum, VLAN, and protocol */
+		ixgbevf_process_skb_fields(rx_ring, rx_desc, skb);
+
+		napi_gro_receive(&q_vector->napi, skb);
+	}
+
+	if (rx_ring->pending >= rx_ring->thresh)
+		failure |= !ixgbevf_xsk_alloc_rx_bufs(rx_ring,
+						      rx_ring->pending);
+
+	/* place incomplete frames back on ring for completion */
+	rx_ring->xsk_xdp_head = head_xdp;
+
+	ixgbevf_xsk_finalize_xdp_napi(&xdp_tx_bulk);
+
+	u64_stats_update_begin(&rx_ring->syncp);
+	rx_ring->stats.packets += total_rx_packets;
+	rx_ring->stats.bytes += total_rx_bytes;
+	u64_stats_update_end(&rx_ring->syncp);
+	q_vector->rx.total_packets += total_rx_packets;
+	q_vector->rx.total_bytes += total_rx_bytes;
+
+	if (likely(!failure))
+		return total_rx_packets;
+
+	if (wake)
+		xsk_set_rx_need_wakeup(rx_ring->xsk_pool);
+
+	return budget;
+}
+
+bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector,
+			      struct ixgbevf_ring *tx_ring, int napi_budget)
+{
+	u32 budget = min_t(u32, napi_budget, tx_ring->thresh);
+
+	if (ring_is_xdp(tx_ring))
+		return libeth_xsk_xmit_do_bulk(tx_ring->xsk_pool, tx_ring,
+					       budget, NULL,
+					       ixgbevf_prep_xdp_sq,
+					       ixgbevf_xsk_xmit_desc,
+					       ixgbevf_xdp_rs_and_bump);
+	else
+		return __libeth_xsk_xmit_do_bulk(tx_ring->xsk_pool, tx_ring,
+						 budget, NULL,
+						 ixgbevf_prep_tx_sq,
+						 ixgbevf_xsk_xmit_desc,
+						 ixgbevf_xdp_rs_and_bump,
+						 ixgbevf_xdp_tx_unprep);
+}
+
+int ixgbevf_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
+{
+	struct ixgbevf_adapter *adapter = netdev_priv(dev);
+	struct ixgbevf_q_vector *q_vector;
+	struct ixgbevf_ring *rx_ring;
+
+	if (unlikely(test_bit(__IXGBEVF_DOWN, &adapter->state)))
+		return -ENETDOWN;
+
+	rx_ring = adapter->rx_ring[queue_id];
+	if (unlikely(!ring_is_xsk(rx_ring)))
+		return -EINVAL;
+
+	q_vector = rx_ring->q_vector;
+	if (!napi_if_scheduled_mark_missed(&q_vector->napi))
+		IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS,
+				BIT(q_vector->v_idx));
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h
new file mode 100644
index 00000000000000..2bb39735b10efd
--- /dev/null
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (C) 2026 Intel Corporation */
+
+#ifndef _IXGBEVF_XSK_H_
+#define _IXGBEVF_XSK_H_
+
+/* Process completions as soon as possible */
+#define IXGBEVF_XSK_TX_CLEAN_THRESH(r)	((r)->count - 1)
+#define IXGBEVF_XSK_MAX_ZC_FRAGS	min(18, MAX_SKB_FRAGS)
+
+int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter,
+			   struct xsk_buff_pool *pool, u16 qid);
+bool ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num);
+void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring);
+u32 ixgbevf_clean_xsk_rx_irq(struct ixgbevf_q_vector *q_vector,
+			     struct ixgbevf_ring *rx_ring, int budget);
+bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector,
+			      struct ixgbevf_ring *tx_ring, int napi_budget);
+int ixgbevf_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags);
+
+#endif /* _IXGBEVF_XSK_H_ */
diff --git a/include/net/libeth/tx.h b/include/net/libeth/tx.h
index c3db5c6f16410c..f3edefe2411da4 100644
--- a/include/net/libeth/tx.h
+++ b/include/net/libeth/tx.h
@@ -74,7 +74,7 @@ struct libeth_sqe {
 	u32				packets;
 	u32				bytes;
 
-	unsigned long			priv;
+	u64				priv;
 } __aligned_largest;
 
 /**
diff --git a/include/net/libeth/xdp.h b/include/net/libeth/xdp.h
index 2e2154ccecae61..92bc82f5dc05e9 100644
--- a/include/net/libeth/xdp.h
+++ b/include/net/libeth/xdp.h
@@ -132,17 +132,18 @@ static inline bool libeth_xdpsq_shared(u32 num)
 /**
  * libeth_xdpsq_id - get XDPSQ index corresponding to this CPU
  * @num: number of active XDPSQs
+ * @force_share: always share queues
  *
  * Helper for libeth_xdp routines, do not use in drivers directly.
  *
  * Return: XDPSQ index needs to be used on this CPU.
  */
-static inline u32 libeth_xdpsq_id(u32 num)
+static inline u32 libeth_xdpsq_id(u32 num, bool force_share)
 {
 	u32 ret = raw_smp_processor_id();
 
-	if (static_branch_unlikely(&libeth_xdpsq_share) &&
-	    libeth_xdpsq_shared(num))
+	if ((static_branch_unlikely(&libeth_xdpsq_share) &&
+	     libeth_xdpsq_shared(num)) || force_share)
 		ret %= num;
 
 	return ret;
@@ -521,7 +522,8 @@ libeth_xdp_tx_xmit_bulk(const struct libeth_xdp_tx_frame *bulk, void *xdpsq,
 			(*fill)(struct libeth_xdp_tx_frame frm, u32 i,
 				const struct libeth_xdpsq *sq, u64 priv),
 			void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i,
-				     const struct libeth_xdpsq *sq, u64 priv))
+				     const struct libeth_xdpsq *sq, u64 priv),
+			void (*unprep)(void *xdpsq, struct libeth_xdpsq *sq))
 {
 	struct libeth_xdpsq sq __uninitialized;
 	u32 this, batched, off = 0;
@@ -579,7 +581,7 @@ libeth_xdp_tx_xmit_bulk(const struct libeth_xdp_tx_frame *bulk, void *xdpsq,
 		*sq.xdp_tx += n;
 
 unlock:
-	libeth_xdpsq_unlock(sq.lock);
+	unprep(xdpsq, &sq);
 
 	return n;
 }
@@ -775,14 +777,15 @@ __libeth_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags,
 				   const struct libeth_xdpsq *sq, u64 priv),
 			   void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i,
 					const struct libeth_xdpsq *sq,
-					u64 priv))
+					u64 priv),
+			   void (*unprep)(void *xdpsq, struct libeth_xdpsq *sq))
 {
 	u32 sent, drops;
 	int err = 0;
 
 	sent = libeth_xdp_tx_xmit_bulk(bq->bulk, bq->xdpsq,
 				       min(bq->count, LIBETH_XDP_TX_BULK),
-				       false, 0, prep, fill, xmit);
+				       false, 0, prep, fill, xmit, unprep);
 	drops = bq->count - sent;
 
 	if (unlikely(drops)) {
@@ -807,9 +810,9 @@ __libeth_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags,
  * Use via LIBETH_XDP_DEFINE_FLUSH_TX() to define an ``XDP_TX`` driver
  * callback.
  */
-#define libeth_xdp_tx_flush_bulk(bq, flags, prep, xmit)			      \
+#define libeth_xdp_tx_flush_bulk(bq, flags, prep, xmit, unprep)		      \
 	__libeth_xdp_tx_flush_bulk(bq, flags, prep, libeth_xdp_tx_fill_buf,   \
-				   xmit)
+				   xmit, unprep)
 
 /* .ndo_xdp_xmit() implementation */
 
@@ -821,7 +824,12 @@ __libeth_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags,
  * @num: number of active XDPSQs (the above array length)
  */
 #define libeth_xdp_xmit_init_bulk(bq, dev, xdpsqs, num)			      \
-	__libeth_xdp_xmit_init_bulk(bq, dev, (xdpsqs)[libeth_xdpsq_id(num)])
+	__libeth_xdp_xmit_init_bulk(bq, dev,				      \
+				   (xdpsqs)[libeth_xdpsq_id(num, false)])
+
+#define libeth_xdp_xmit_init_bulk_shared(bq, dev, xdpsqs, num)		      \
+	__libeth_xdp_xmit_init_bulk(bq, dev,				      \
+				   (xdpsqs)[libeth_xdpsq_id(num, true)])
 
 static inline void __libeth_xdp_xmit_init_bulk(struct libeth_xdp_tx_bulk *bq,
 					       struct net_device *dev,
@@ -1017,6 +1025,12 @@ libeth_xdp_xmit_fill_buf(struct libeth_xdp_tx_frame frm, u32 i,
 	return desc;
 }
 
+static inline void libeth_xdp_tx_unprep(void *xdpsq __always_unused,
+					struct libeth_xdpsq *sq)
+{
+	libeth_xdpsq_unlock(sq->lock);
+}
+
 /**
  * libeth_xdp_xmit_flush_bulk - wrapper to define flush of one XDP xmit bulk
  * @bq: bulk to flush
@@ -1027,9 +1041,10 @@ libeth_xdp_xmit_fill_buf(struct libeth_xdp_tx_frame frm, u32 i,
  * Use via LIBETH_XDP_DEFINE_FLUSH_XMIT() to define an XDP xmit driver
  * callback.
  */
-#define libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit)		      \
+#define libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit, unprep)		      \
 	__libeth_xdp_tx_flush_bulk(bq, (flags) | LIBETH_XDP_TX_NDO, prep,     \
-				   libeth_xdp_xmit_fill_buf, xmit)
+				   libeth_xdp_xmit_fill_buf, xmit,	      \
+				   unprep)
 
 u32 libeth_xdp_xmit_return_bulk(const struct libeth_xdp_tx_frame *bq,
 				u32 count, const struct net_device *dev);
@@ -1139,10 +1154,15 @@ __libeth_xdp_xmit_do_bulk(struct libeth_xdp_tx_bulk *bq,
  * Do not use for XSk, it has its own optimized helper.
  */
 #define libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num)		      \
-	__libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, false,	      \
+	__libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, false, false,   \
 				  __UNIQUE_ID(bq_), __UNIQUE_ID(nqs_))
 
-#define __libeth_xdp_tx_init_bulk(bq, pr, d, xdpsqs, num, xsk, ub, un) do {   \
+#define libeth_xdp_tx_init_bulk_shared(bq, prog, dev, xdpsqs, num)	      \
+	__libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, false, true,    \
+				  __UNIQUE_ID(bq_), __UNIQUE_ID(nqs_))
+
+#define __libeth_xdp_tx_init_bulk(bq, pr, d, xdpsqs, num, xsk, fs, ub, un)    \
+do {									      \
 	typeof(bq) ub = (bq);						      \
 	u32 un = (num);							      \
 									      \
@@ -1151,7 +1171,7 @@ __libeth_xdp_xmit_do_bulk(struct libeth_xdp_tx_bulk *bq,
 	if (un || (xsk)) {						      \
 		ub->prog = rcu_dereference(pr);				      \
 		ub->dev = (d);						      \
-		ub->xdpsq = (xdpsqs)[libeth_xdpsq_id(un)];		      \
+		ub->xdpsq = (xdpsqs)[libeth_xdpsq_id(un, fs)];		      \
 	} else {							      \
 		ub->prog = NULL;					      \
 	}								      \
@@ -1611,12 +1631,13 @@ void name(struct work_struct *work)					      \
  * @xmit: driver callback to write a HW Tx descriptor
  */
 #define LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit)			      \
-	__LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, xdp)
+	__LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, libeth_xdp_tx_unprep,  \
+				     xdp)
 
-#define __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, pfx)		      \
+#define __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, unprep, pfx)	      \
 bool name(struct libeth_xdp_tx_bulk *bq, u32 flags)			      \
 {									      \
-	return libeth_##pfx##_tx_flush_bulk(bq, flags, prep, xmit);	      \
+	return libeth_##pfx##_tx_flush_bulk(bq, flags, prep, xmit, unprep);   \
 }
 
 /**
@@ -1626,9 +1647,12 @@ bool name(struct libeth_xdp_tx_bulk *bq, u32 flags)			      \
  * @xmit: driver callback to write a HW Tx descriptor
  */
 #define LIBETH_XDP_DEFINE_FLUSH_XMIT(name, prep, xmit)			      \
+	__LIBETH_XDP_DEFINE_FLUSH_XMIT(name, prep, xmit, libeth_xdp_tx_unprep)
+
+#define __LIBETH_XDP_DEFINE_FLUSH_XMIT(name, prep, xmit, unprep)	      \
 bool name(struct libeth_xdp_tx_bulk *bq, u32 flags)			      \
 {									      \
-	return libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit);	      \
+	return libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit, unprep);     \
 }
 
 /**
diff --git a/include/net/libeth/xsk.h b/include/net/libeth/xsk.h
index 82b5d21aae8784..fa66b740994ca4 100644
--- a/include/net/libeth/xsk.h
+++ b/include/net/libeth/xsk.h
@@ -146,9 +146,10 @@ libeth_xsk_tx_fill_buf(struct libeth_xdp_tx_frame frm, u32 i,
  * Use via LIBETH_XSK_DEFINE_FLUSH_TX() to define an XSk ``XDP_TX`` driver
  * callback.
  */
-#define libeth_xsk_tx_flush_bulk(bq, flags, prep, xmit)			     \
+#define libeth_xsk_tx_flush_bulk(bq, flags, prep, xmit, unprep)		     \
 	__libeth_xdp_tx_flush_bulk(bq, (flags) | LIBETH_XDP_TX_XSK, prep,    \
-				   libeth_xsk_tx_fill_buf, xmit)
+				   libeth_xsk_tx_fill_buf, xmit,	     \
+				   unprep)
 
 /* XSk TMO */
 
@@ -282,12 +283,13 @@ libeth_xsk_xmit_fill_buf(struct libeth_xdp_tx_frame frm, u32 i,
  * Return: false if @budget was exhausted, true otherwise.
  */
 static __always_inline bool
-libeth_xsk_xmit_do_bulk(struct xsk_buff_pool *pool, void *xdpsq, u32 budget,
-			const struct xsk_tx_metadata_ops *tmo,
-			u32 (*prep)(void *xdpsq, struct libeth_xdpsq *sq),
-			void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i,
-				     const struct libeth_xdpsq *sq, u64 priv),
-			void (*finalize)(void *xdpsq, bool sent, bool flush))
+__libeth_xsk_xmit_do_bulk(struct xsk_buff_pool *pool, void *xdpsq, u32 budget,
+			  const struct xsk_tx_metadata_ops *tmo,
+			  u32 (*prep)(void *xdpsq, struct libeth_xdpsq *sq),
+			  void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i,
+				       const struct libeth_xdpsq *sq, u64 priv),
+			  void (*finalize)(void *xdpsq, bool sent, bool flush),
+			  void (*unprep)(void *xdpsq, struct libeth_xdpsq *sq))
 {
 	const struct libeth_xdp_tx_frame *bulk;
 	bool wake;
@@ -298,11 +300,14 @@ libeth_xsk_xmit_do_bulk(struct xsk_buff_pool *pool, void *xdpsq, u32 budget,
 		xsk_clear_tx_need_wakeup(pool);
 
 	n = xsk_tx_peek_release_desc_batch(pool, budget);
+	if (unlikely(!n))
+		return true;
 	bulk = container_of(&pool->tx_descs[0], typeof(*bulk), desc);
 
 	libeth_xdp_tx_xmit_bulk(bulk, xdpsq, n, true,
 				libeth_xdp_ptr_to_priv(tmo), prep,
-				libeth_xsk_xmit_fill_buf, xmit);
+				libeth_xsk_xmit_fill_buf, xmit,
+				unprep);
 	finalize(xdpsq, n, true);
 
 	if (wake)
@@ -311,6 +316,10 @@ libeth_xsk_xmit_do_bulk(struct xsk_buff_pool *pool, void *xdpsq, u32 budget,
 	return n < budget;
 }
 
+#define libeth_xsk_xmit_do_bulk(pool, xdpsq, budget, tmo, prep, xmit, finalize)\
+	__libeth_xsk_xmit_do_bulk(pool, xdpsq, budget, tmo, prep, xmit,	       \
+				  finalize, libeth_xdp_tx_unprep)
+
 /* Rx polling path */
 
 /**
@@ -327,7 +336,11 @@ libeth_xsk_xmit_do_bulk(struct xsk_buff_pool *pool, void *xdpsq, u32 budget,
  * when hitting this path.
  */
 #define libeth_xsk_tx_init_bulk(bq, prog, dev, xdpsqs, num)		     \
-	__libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, true,	     \
+	__libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, true, false,   \
+				  __UNIQUE_ID(bq_), __UNIQUE_ID(nqs_))
+
+#define libeth_xsk_tx_init_bulk_shared(bq, prog, dev, xdpsqs, num)	     \
+	__libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, true, true,    \
 				  __UNIQUE_ID(bq_), __UNIQUE_ID(nqs_))
 
 struct libeth_xdp_buff *libeth_xsk_buff_add_frag(struct libeth_xdp_buff *head,
@@ -546,7 +559,8 @@ __libeth_xsk_run_pass(struct libeth_xdp_buff *xdp,
  * @xmit: driver callback to write a HW Tx descriptor
  */
 #define LIBETH_XSK_DEFINE_FLUSH_TX(name, prep, xmit)			     \
-	__LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, xsk)
+	__LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, libeth_xdp_tx_unprep, \
+				     xsk)
 
 /**
  * LIBETH_XSK_DEFINE_RUN_PROG - define a driver XDP program run function