ixgbevf XDP shared queues

drivers/net/ethernet/intel/ixgbevf/Makefile

@@ -6,5 +6,5 @@
 
 obj-$(CONFIG_IXGBEVF) += ixgbevf.o
 
-ixgbevf-y := vf.o mbx.o ethtool.o ixgbevf_main.o
+ixgbevf-y := vf.o mbx.o ethtool.o ixgbevf_main.o ixgbevf_xsk.o
 ixgbevf-$(CONFIG_IXGBEVF_IPSEC) += ipsec.o

drivers/net/ethernet/intel/ixgbevf/ethtool.c

@@ -916,6 +916,75 @@ static int ixgbevf_get_rxfh(struct net_device *netdev,
 	return err;
 }
 
+static void ixgbevf_get_reported_q_num(u32 rx, u32 tx, u32 *dst_combined,
+				       u32 *dst_rx, u32 *dst_tx)
+{
+	rx = min_t(u32, rx, num_online_cpus());
+	*dst_combined = min_t(u32, rx, tx);
+	if (rx > tx) {
+		*dst_rx = rx - tx;
+		*dst_tx = 0;
+	} else {
+		*dst_tx = tx - rx;
+		*dst_rx = 0;
+	}
+}
+
+static void ixgbevf_get_channels(struct net_device *netdev,
+				 struct ethtool_channels *ch)
+{
+	struct ixgbevf_adapter *adapter = netdev_priv(netdev);
+
+	ixgbevf_get_reported_q_num(adapter->num_rx_queues,
+				   adapter->num_tx_queues,
+				   &ch->combined_count, &ch->rx_count,
+				   &ch->tx_count);
+
+	ixgbevf_get_reported_q_num(adapter->q_caps.max_rxqs,
+				   adapter->q_caps.max_txqs,
+				   &ch->max_combined, &ch->max_rx,
+				   &ch->max_tx);
+
+	ch->max_other = NON_Q_VECTORS;
+	ch->other_count = NON_Q_VECTORS;
+}
+
+static int ixgbevf_set_channels(struct net_device *netdev,
+				struct ethtool_channels *ch)
+{
+	struct ixgbevf_adapter *adapter = netdev_priv(netdev);
+	u32 num_req = ch->combined_count;
+	bool was_up;
+	int err = 0;
+
+	/* Do not change queue number if DCB is enabled */
+	if (adapter->q_caps.num_tcs > 1)
+		return -EOPNOTSUPP;
+
+	if (ch->rx_count || ch->tx_count || ch->other_count != NON_Q_VECTORS)
+		return -EINVAL;
+
+	if (num_req == adapter->num_rx_queues &&
+	    num_req == adapter->num_tx_queues)
+		return 0;
+
+	adapter->num_req_qpairs = num_req;
+
+	was_up = netif_running(netdev);
+	if (was_up)
+		ixgbevf_close(netdev);
+
+	ixgbevf_clear_interrupt_scheme(adapter);
+	err = ixgbevf_init_interrupt_scheme(adapter);
+	if (err)
+		return err;
+
+	if (was_up)
+		ixgbevf_open(netdev);
+
+	return 0;
+}
+
 static const struct ethtool_ops ixgbevf_ethtool_ops = {
 	.supported_coalesce_params = ETHTOOL_COALESCE_USECS,
 	.get_drvinfo		= ixgbevf_get_drvinfo,
@@ -938,6 +1007,8 @@ static const struct ethtool_ops ixgbevf_ethtool_ops = {
 	.get_rxfh_key_size	= ixgbevf_get_rxfh_key_size,
 	.get_rxfh		= ixgbevf_get_rxfh,
 	.get_link_ksettings	= ixgbevf_get_link_ksettings,
+	.get_channels		= ixgbevf_get_channels,
+	.set_channels		= ixgbevf_set_channels,
 };
 
 void ixgbevf_set_ethtool_ops(struct net_device *netdev)

drivers/net/ethernet/intel/ixgbevf/ipsec.c

@@ -450,11 +450,11 @@ static const struct xfrmdev_ops ixgbevf_xfrmdev_ops = {
  * @first: current data packet
  * @itd: ipsec Tx data for later use in building context descriptor
  **/
-int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring,
-		     struct ixgbevf_tx_buffer *first,
+int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring, struct libeth_sqe *first,
 		     struct ixgbevf_ipsec_tx_data *itd)
 {
 	struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev);
+	struct ixgbevf_skb_sqe_priv *sqe_priv = (void *)&first->priv;
 	struct ixgbevf_ipsec *ipsec = adapter->ipsec;
 	struct xfrm_state *xs;
 	struct sec_path *sp;
@@ -491,12 +491,12 @@ int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring,
 
 	itd->pfsa = tsa->pfsa - IXGBE_IPSEC_BASE_TX_INDEX;
 
-	first->tx_flags |= IXGBE_TX_FLAGS_IPSEC | IXGBE_TX_FLAGS_CSUM;
+	sqe_priv->tx_flags |= IXGBE_TX_FLAGS_IPSEC | IXGBE_TX_FLAGS_CSUM;
 
 	if (xs->id.proto == IPPROTO_ESP) {
 		itd->flags |= IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP |
 			      IXGBE_ADVTXD_TUCMD_L4T_TCP;
-		if (first->protocol == htons(ETH_P_IP))
+		if (sqe_priv->protocol == htons(ETH_P_IP))
 			itd->flags |= IXGBE_ADVTXD_TUCMD_IPV4;
 
 		/* The actual trailer length is authlen (16 bytes) plus

drivers/net/ethernet/intel/ixgbevf/ixgbevf.h

@@ -11,7 +11,7 @@
 #include <linux/netdevice.h>
 #include <linux/if_vlan.h>
 #include <linux/u64_stats_sync.h>
-#include <net/libeth/types.h>
+#include <net/libeth/tx.h>
 #include <net/xdp.h>
 
 #include "vf.h"
@@ -24,24 +24,12 @@
 #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)
 #define DESC_NEEDED (MAX_SKB_FRAGS + 4)
 
-/* wrapper around a pointer to a socket buffer,
- * so a DMA handle can be stored along with the buffer
- */
-struct ixgbevf_tx_buffer {
-	union ixgbe_adv_tx_desc *next_to_watch;
-	unsigned long time_stamp;
-	union {
-		struct sk_buff *skb;
-		/* XDP uses address ptr on irq_clean */
-		void *data;
-	};
-	unsigned int bytecount;
-	unsigned short gso_segs;
-	__be16 protocol;
-	DEFINE_DMA_UNMAP_ADDR(dma);
-	DEFINE_DMA_UNMAP_LEN(len);
+struct ixgbevf_skb_sqe_priv {
 	u32 tx_flags;
+	__be16 protocol;
 };
+static_assert(sizeof(struct ixgbevf_skb_sqe_priv) <=
+	      sizeof(typeof_member(struct libeth_sqe, priv)));
 
 struct ixgbevf_stats {
 	u64 packets;
@@ -66,6 +54,7 @@ enum ixgbevf_ring_state_t {
 	__IXGBEVF_HANG_CHECK_ARMED,
 	__IXGBEVF_TX_XDP_RING,
 	__IXGBEVF_TX_XDP_RING_PRIMED,
+	__IXGBEVF_RXTX_XSK_RING,
 };
 
 #define ring_is_xdp(ring) \
@@ -75,6 +64,13 @@ enum ixgbevf_ring_state_t {
 #define clear_ring_xdp(ring) \
 		clear_bit(__IXGBEVF_TX_XDP_RING, &(ring)->state)
 
+#define ring_is_xsk(ring) \
+		test_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state)
+#define set_ring_xsk(ring) \
+		set_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state)
+#define clear_ring_xsk(ring) \
+		clear_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state)
+
 struct ixgbevf_ring {
 	struct ixgbevf_ring *next;
 	struct ixgbevf_q_vector *q_vector;	/* backpointer to q_vector */
@@ -85,22 +81,22 @@ struct ixgbevf_ring {
 		struct device *dev;	/* Tx ring */
 	};
 	void *desc;			/* descriptor ring memory */
-	union {
-		u32 truesize;		/* Rx buffer full size */
-		u32 pending;		/* Sent-not-completed descriptors */
-	};
+	u32 truesize;			/* Rx buffer full size */
 	u32 hdr_truesize;		/* Rx header buffer full size */
 	u16 count;			/* amount of descriptors */
 	u16 next_to_clean;
 	u32 next_to_use;
+	u32 pending;			/* Sent-not-completed descriptors */
 
 	union {
 		struct libeth_fqe *rx_fqes;
-		struct ixgbevf_tx_buffer *tx_buffer_info;
+		struct libeth_xdp_buff	**xsk_fqes;
+		struct libeth_sqe *tx_sqes;
 		struct libeth_sqe *xdp_sqes;
 	};
 	struct libeth_xdpsq_lock xdpq_lock;
 	u32 cached_ntu;
+	u32 thresh;
 	unsigned long state;
 	struct ixgbevf_stats stats;
 	struct u64_stats_sync syncp;
@@ -121,8 +117,10 @@ struct ixgbevf_ring {
 	int queue_index; /* needed for multiqueue queue management */
 	u32 rx_buf_len;
 	struct libeth_xdp_buff_stash xdp_stash;
+	struct libeth_xdp_buff *xsk_xdp_head;
 	unsigned int dma_size;		/* length in bytes */
 	dma_addr_t dma;			/* phys. address of descriptor ring */
+	struct xsk_buff_pool *xsk_pool; /* AF_XDP ZC rings */
 } ____cacheline_internodealigned_in_smp;
 
 /* How many Rx Buffers do we bundle into one write to the hardware ? */
@@ -235,6 +233,14 @@ static inline u16 ixgbevf_desc_unused(struct ixgbevf_ring *ring)
 	return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - 1;
 }
 
+static inline u16 ixgbevf_desc_used(struct ixgbevf_ring *ring)
+{
+	u16 ntc = ring->next_to_clean;
+	u16 ntu = ring->next_to_use;
+
+	return ((ntu >= ntc) ? 0 : ring->count) + ntu - ntc;
+}
+
 static inline void ixgbevf_write_tail(struct ixgbevf_ring *ring, u32 value)
 {
 	writel(value, ring->tail);
@@ -260,6 +266,15 @@ static inline void ixgbevf_write_tail(struct ixgbevf_ring *ring, u32 value)
 #define IXGBEVF_RX_DMA_ATTR \
 	(DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
 
+struct ixgbevf_q_caps {
+	u32 min_rxqs;
+	u32 max_rxqs;
+	u32 min_txqs;
+	u32 max_txqs;
+	u32 num_tcs;
+	u32 def_tc;
+};
+
 /* board specific private data structure */
 struct ixgbevf_adapter {
 	/* this field must be first, see ixgbevf_process_skb_fields */
@@ -336,6 +351,8 @@ struct ixgbevf_adapter {
 	u8 rss_indir_tbl[IXGBEVF_X550_VFRETA_SIZE];
 	u32 flags;
 	bool link_state;
+	struct ixgbevf_q_caps q_caps;
+	u32 num_req_qpairs;
 
 #ifdef CONFIG_XFRM
 	struct ixgbevf_ipsec *ipsec;
@@ -399,15 +416,31 @@ int ixgbevf_open(struct net_device *netdev);
 int ixgbevf_close(struct net_device *netdev);
 void ixgbevf_up(struct ixgbevf_adapter *adapter);
 void ixgbevf_down(struct ixgbevf_adapter *adapter);
+void ixgbevf_flush_tx_queue(struct ixgbevf_ring *ring);
+void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter,
+			      struct ixgbevf_ring *ring);
+void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
+				  struct ixgbevf_ring *ring);
 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter);
 void ixgbevf_reset(struct ixgbevf_adapter *adapter);
 void ixgbevf_set_ethtool_ops(struct net_device *netdev);
 int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter,
 			       struct ixgbevf_ring *rx_ring);
+void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter);
+void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
+			       struct ixgbevf_ring *ring);
 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *);
+void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
+			       struct ixgbevf_ring *ring);
 void ixgbevf_free_rx_resources(struct ixgbevf_ring *);
+void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring);
+void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring);
 void ixgbevf_free_tx_resources(struct ixgbevf_ring *);
+void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring);
+void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring);
 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter);
+int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter);
+void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter);
 int ethtool_ioctl(struct ifreq *ifr);
 
 extern void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector);
@@ -419,8 +452,7 @@ void ixgbevf_ipsec_restore(struct ixgbevf_adapter *adapter);
 void ixgbevf_ipsec_rx(struct ixgbevf_ring *rx_ring,
 		      union ixgbe_adv_rx_desc *rx_desc,
 		      struct sk_buff *skb);
-int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring,
-		     struct ixgbevf_tx_buffer *first,
+int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring, struct libeth_sqe *first,
 		     struct ixgbevf_ipsec_tx_data *itd);
 #else
 static inline void ixgbevf_init_ipsec_offload(struct ixgbevf_adapter *adapter)