|
@@ -87,16 +87,31 @@ void nfp_net_get_fw_version(struct nfp_net_fw_version *fw_ver,
|
|
|
|
|
|
static dma_addr_t nfp_net_dma_map_rx(struct nfp_net_dp *dp, void *frag)
|
|
|
{
|
|
|
- return dma_map_single(dp->dev, frag + NFP_NET_RX_BUF_HEADROOM,
|
|
|
- dp->fl_bufsz - NFP_NET_RX_BUF_NON_DATA,
|
|
|
- dp->rx_dma_dir);
|
|
|
+ return dma_map_single_attrs(dp->dev, frag + NFP_NET_RX_BUF_HEADROOM,
|
|
|
+ dp->fl_bufsz - NFP_NET_RX_BUF_NON_DATA,
|
|
|
+ dp->rx_dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
|
|
|
+}
|
|
|
+
|
|
|
+static void
|
|
|
+nfp_net_dma_sync_dev_rx(const struct nfp_net_dp *dp, dma_addr_t dma_addr)
|
|
|
+{
|
|
|
+ dma_sync_single_for_device(dp->dev, dma_addr,
|
|
|
+ dp->fl_bufsz - NFP_NET_RX_BUF_NON_DATA,
|
|
|
+ dp->rx_dma_dir);
|
|
|
}
|
|
|
|
|
|
static void nfp_net_dma_unmap_rx(struct nfp_net_dp *dp, dma_addr_t dma_addr)
|
|
|
{
|
|
|
- dma_unmap_single(dp->dev, dma_addr,
|
|
|
- dp->fl_bufsz - NFP_NET_RX_BUF_NON_DATA,
|
|
|
- dp->rx_dma_dir);
|
|
|
+ dma_unmap_single_attrs(dp->dev, dma_addr,
|
|
|
+ dp->fl_bufsz - NFP_NET_RX_BUF_NON_DATA,
|
|
|
+ dp->rx_dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
|
|
|
+}
|
|
|
+
|
|
|
+static void nfp_net_dma_sync_cpu_rx(struct nfp_net_dp *dp, dma_addr_t dma_addr,
|
|
|
+ unsigned int len)
|
|
|
+{
|
|
|
+ dma_sync_single_for_cpu(dp->dev, dma_addr - NFP_NET_RX_BUF_HEADROOM,
|
|
|
+ len, dp->rx_dma_dir);
|
|
|
}
|
|
|
|
|
|
/* Firmware reconfig
|
|
@@ -1208,6 +1223,8 @@ static void nfp_net_rx_give_one(const struct nfp_net_dp *dp,
|
|
|
|
|
|
wr_idx = rx_ring->wr_p & (rx_ring->cnt - 1);
|
|
|
|
|
|
+ nfp_net_dma_sync_dev_rx(dp, dma_addr);
|
|
|
+
|
|
|
/* Stash SKB and DMA address away */
|
|
|
rx_ring->rxbufs[wr_idx].frag = frag;
|
|
|
rx_ring->rxbufs[wr_idx].dma_addr = dma_addr;
|
|
@@ -1385,8 +1402,9 @@ static void nfp_net_rx_csum(struct nfp_net_dp *dp,
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-static void nfp_net_set_hash(struct net_device *netdev, struct sk_buff *skb,
|
|
|
- unsigned int type, __be32 *hash)
|
|
|
+static void
|
|
|
+nfp_net_set_hash(struct net_device *netdev, struct nfp_meta_parsed *meta,
|
|
|
+ unsigned int type, __be32 *hash)
|
|
|
{
|
|
|
if (!(netdev->features & NETIF_F_RXHASH))
|
|
|
return;
|
|
@@ -1395,16 +1413,18 @@ static void nfp_net_set_hash(struct net_device *netdev, struct sk_buff *skb,
|
|
|
case NFP_NET_RSS_IPV4:
|
|
|
case NFP_NET_RSS_IPV6:
|
|
|
case NFP_NET_RSS_IPV6_EX:
|
|
|
- skb_set_hash(skb, get_unaligned_be32(hash), PKT_HASH_TYPE_L3);
|
|
|
+ meta->hash_type = PKT_HASH_TYPE_L3;
|
|
|
break;
|
|
|
default:
|
|
|
- skb_set_hash(skb, get_unaligned_be32(hash), PKT_HASH_TYPE_L4);
|
|
|
+ meta->hash_type = PKT_HASH_TYPE_L4;
|
|
|
break;
|
|
|
}
|
|
|
+
|
|
|
+ meta->hash = get_unaligned_be32(hash);
|
|
|
}
|
|
|
|
|
|
static void
|
|
|
-nfp_net_set_hash_desc(struct net_device *netdev, struct sk_buff *skb,
|
|
|
+nfp_net_set_hash_desc(struct net_device *netdev, struct nfp_meta_parsed *meta,
|
|
|
void *data, struct nfp_net_rx_desc *rxd)
|
|
|
{
|
|
|
struct nfp_net_rx_hash *rx_hash = data;
|
|
@@ -1412,12 +1432,12 @@ nfp_net_set_hash_desc(struct net_device *netdev, struct sk_buff *skb,
|
|
|
if (!(rxd->rxd.flags & PCIE_DESC_RX_RSS))
|
|
|
return;
|
|
|
|
|
|
- nfp_net_set_hash(netdev, skb, get_unaligned_be32(&rx_hash->hash_type),
|
|
|
+ nfp_net_set_hash(netdev, meta, get_unaligned_be32(&rx_hash->hash_type),
|
|
|
&rx_hash->hash);
|
|
|
}
|
|
|
|
|
|
static void *
|
|
|
-nfp_net_parse_meta(struct net_device *netdev, struct sk_buff *skb,
|
|
|
+nfp_net_parse_meta(struct net_device *netdev, struct nfp_meta_parsed *meta,
|
|
|
void *data, int meta_len)
|
|
|
{
|
|
|
u32 meta_info;
|
|
@@ -1429,13 +1449,13 @@ nfp_net_parse_meta(struct net_device *netdev, struct sk_buff *skb,
|
|
|
switch (meta_info & NFP_NET_META_FIELD_MASK) {
|
|
|
case NFP_NET_META_HASH:
|
|
|
meta_info >>= NFP_NET_META_FIELD_SIZE;
|
|
|
- nfp_net_set_hash(netdev, skb,
|
|
|
+ nfp_net_set_hash(netdev, meta,
|
|
|
meta_info & NFP_NET_META_FIELD_MASK,
|
|
|
(__be32 *)data);
|
|
|
data += 4;
|
|
|
break;
|
|
|
case NFP_NET_META_MARK:
|
|
|
- skb->mark = get_unaligned_be32(data);
|
|
|
+ meta->mark = get_unaligned_be32(data);
|
|
|
data += 4;
|
|
|
break;
|
|
|
default:
|
|
@@ -1569,13 +1589,12 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget)
|
|
|
tx_ring = r_vec->xdp_ring;
|
|
|
|
|
|
while (pkts_polled < budget) {
|
|
|
- unsigned int meta_len, data_len, data_off, pkt_len;
|
|
|
- u8 meta_prepend[NFP_NET_MAX_PREPEND];
|
|
|
+ unsigned int meta_len, data_len, meta_off, pkt_len, pkt_off;
|
|
|
struct nfp_net_rx_buf *rxbuf;
|
|
|
struct nfp_net_rx_desc *rxd;
|
|
|
+ struct nfp_meta_parsed meta;
|
|
|
dma_addr_t new_dma_addr;
|
|
|
void *new_frag;
|
|
|
- u8 *meta;
|
|
|
|
|
|
idx = rx_ring->rd_p & (rx_ring->cnt - 1);
|
|
|
|
|
@@ -1588,6 +1607,8 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget)
|
|
|
*/
|
|
|
dma_rmb();
|
|
|
|
|
|
+ memset(&meta, 0, sizeof(meta));
|
|
|
+
|
|
|
rx_ring->rd_p++;
|
|
|
pkts_polled++;
|
|
|
|
|
@@ -1608,11 +1629,12 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget)
|
|
|
data_len = le16_to_cpu(rxd->rxd.data_len);
|
|
|
pkt_len = data_len - meta_len;
|
|
|
|
|
|
+ pkt_off = NFP_NET_RX_BUF_HEADROOM + dp->rx_dma_off;
|
|
|
if (dp->rx_offset == NFP_NET_CFG_RX_OFFSET_DYNAMIC)
|
|
|
- data_off = NFP_NET_RX_BUF_HEADROOM + meta_len;
|
|
|
+ pkt_off += meta_len;
|
|
|
else
|
|
|
- data_off = NFP_NET_RX_BUF_HEADROOM + dp->rx_offset;
|
|
|
- data_off += dp->rx_dma_off;
|
|
|
+ pkt_off += dp->rx_offset;
|
|
|
+ meta_off = pkt_off - meta_len;
|
|
|
|
|
|
/* Stats update */
|
|
|
u64_stats_update_begin(&r_vec->rx_sync);
|
|
@@ -1620,9 +1642,6 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget)
|
|
|
r_vec->rx_bytes += pkt_len;
|
|
|
u64_stats_update_end(&r_vec->rx_sync);
|
|
|
|
|
|
- /* Pointer to start of metadata */
|
|
|
- meta = rxbuf->frag + data_off - meta_len;
|
|
|
-
|
|
|
if (unlikely(meta_len > NFP_NET_MAX_PREPEND ||
|
|
|
(dp->rx_offset && meta_len > dp->rx_offset))) {
|
|
|
nn_dp_warn(dp, "oversized RX packet metadata %u\n",
|
|
@@ -1631,6 +1650,26 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget)
|
|
|
continue;
|
|
|
}
|
|
|
|
|
|
+ nfp_net_dma_sync_cpu_rx(dp, rxbuf->dma_addr + meta_off,
|
|
|
+ data_len);
|
|
|
+
|
|
|
+ if (!dp->chained_metadata_format) {
|
|
|
+ nfp_net_set_hash_desc(dp->netdev, &meta,
|
|
|
+ rxbuf->frag + meta_off, rxd);
|
|
|
+ } else if (meta_len) {
|
|
|
+ void *end;
|
|
|
+
|
|
|
+ end = nfp_net_parse_meta(dp->netdev, &meta,
|
|
|
+ rxbuf->frag + meta_off,
|
|
|
+ meta_len);
|
|
|
+ if (unlikely(end != rxbuf->frag + pkt_off)) {
|
|
|
+ nn_dp_warn(dp, "invalid RX packet metadata\n");
|
|
|
+ nfp_net_rx_drop(dp, r_vec, rx_ring, rxbuf,
|
|
|
+ NULL);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
if (xdp_prog && !(rxd->rxd.flags & PCIE_DESC_RX_BPF &&
|
|
|
dp->bpf_offload_xdp)) {
|
|
|
unsigned int dma_off;
|
|
@@ -1638,24 +1677,14 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget)
|
|
|
int act;
|
|
|
|
|
|
hard_start = rxbuf->frag + NFP_NET_RX_BUF_HEADROOM;
|
|
|
- dma_off = data_off - NFP_NET_RX_BUF_HEADROOM;
|
|
|
- dma_sync_single_for_cpu(dp->dev, rxbuf->dma_addr,
|
|
|
- dma_off + pkt_len,
|
|
|
- DMA_BIDIRECTIONAL);
|
|
|
-
|
|
|
- /* Move prepend out of the way */
|
|
|
- if (xdp_prog->xdp_adjust_head) {
|
|
|
- memcpy(meta_prepend, meta, meta_len);
|
|
|
- meta = meta_prepend;
|
|
|
- }
|
|
|
|
|
|
act = nfp_net_run_xdp(xdp_prog, rxbuf->frag, hard_start,
|
|
|
- &data_off, &pkt_len);
|
|
|
+ &pkt_off, &pkt_len);
|
|
|
switch (act) {
|
|
|
case XDP_PASS:
|
|
|
break;
|
|
|
case XDP_TX:
|
|
|
- dma_off = data_off - NFP_NET_RX_BUF_HEADROOM;
|
|
|
+ dma_off = pkt_off - NFP_NET_RX_BUF_HEADROOM;
|
|
|
if (unlikely(!nfp_net_tx_xdp_buf(dp, rx_ring,
|
|
|
tx_ring, rxbuf,
|
|
|
dma_off,
|
|
@@ -1689,22 +1718,11 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget)
|
|
|
|
|
|
nfp_net_rx_give_one(dp, rx_ring, new_frag, new_dma_addr);
|
|
|
|
|
|
- skb_reserve(skb, data_off);
|
|
|
+ skb_reserve(skb, pkt_off);
|
|
|
skb_put(skb, pkt_len);
|
|
|
|
|
|
- if (!dp->chained_metadata_format) {
|
|
|
- nfp_net_set_hash_desc(dp->netdev, skb, meta, rxd);
|
|
|
- } else if (meta_len) {
|
|
|
- void *end;
|
|
|
-
|
|
|
- end = nfp_net_parse_meta(dp->netdev, skb, meta,
|
|
|
- meta_len);
|
|
|
- if (unlikely(end != meta + meta_len)) {
|
|
|
- nn_dp_warn(dp, "invalid RX packet metadata\n");
|
|
|
- nfp_net_rx_drop(dp, r_vec, rx_ring, NULL, skb);
|
|
|
- continue;
|
|
|
- }
|
|
|
- }
|
|
|
+ skb->mark = meta.mark;
|
|
|
+ skb_set_hash(skb, meta.hash, meta.hash_type);
|
|
|
|
|
|
skb_record_rx_queue(skb, rx_ring->idx);
|
|
|
skb->protocol = eth_type_trans(skb, dp->netdev);
|
|
@@ -2147,7 +2165,7 @@ nfp_net_tx_ring_hw_cfg_write(struct nfp_net *nn,
|
|
|
*/
|
|
|
static int nfp_net_set_config_and_enable(struct nfp_net *nn)
|
|
|
{
|
|
|
- u32 new_ctrl, update = 0;
|
|
|
+ u32 bufsz, new_ctrl, update = 0;
|
|
|
unsigned int r;
|
|
|
int err;
|
|
|
|
|
@@ -2181,8 +2199,9 @@ static int nfp_net_set_config_and_enable(struct nfp_net *nn)
|
|
|
nfp_net_write_mac_addr(nn);
|
|
|
|
|
|
nn_writel(nn, NFP_NET_CFG_MTU, nn->dp.netdev->mtu);
|
|
|
- nn_writel(nn, NFP_NET_CFG_FLBUFSZ,
|
|
|
- nn->dp.fl_bufsz - NFP_NET_RX_BUF_NON_DATA);
|
|
|
+
|
|
|
+ bufsz = nn->dp.fl_bufsz - nn->dp.rx_dma_off - NFP_NET_RX_BUF_NON_DATA;
|
|
|
+ nn_writel(nn, NFP_NET_CFG_FLBUFSZ, bufsz);
|
|
|
|
|
|
/* Enable device */
|
|
|
new_ctrl |= NFP_NET_CFG_CTRL_ENABLE;
|