|
@@ -26,6 +26,13 @@
|
|
|
#include <linux/skbuff.h>
|
|
|
#include <linux/smp.h>
|
|
|
#include <asm/byteorder.h>
|
|
|
+#include <linux/ip.h>
|
|
|
+#include <linux/tcp.h>
|
|
|
+#include <linux/sctp.h>
|
|
|
+#include <linux/ipv6.h>
|
|
|
+#include <net/ipv6.h>
|
|
|
+#include <net/checksum.h>
|
|
|
+#include <net/ip6_checksum.h>
|
|
|
|
|
|
#include "hinic_common.h"
|
|
|
#include "hinic_hw_if.h"
|
|
@@ -45,9 +52,31 @@
|
|
|
#define CI_UPDATE_NO_PENDING 0
|
|
|
#define CI_UPDATE_NO_COALESC 0
|
|
|
|
|
|
-#define HW_CONS_IDX(sq) be16_to_cpu(*(u16 *)((sq)->hw_ci_addr))
|
|
|
+#define HW_CONS_IDX(sq) be16_to_cpu(*(u16 *)((sq)->hw_ci_addr))
|
|
|
|
|
|
-#define MIN_SKB_LEN 64
|
|
|
+#define MIN_SKB_LEN 17
|
|
|
+
|
|
|
+#define MAX_PAYLOAD_OFFSET 221
|
|
|
+#define TRANSPORT_OFFSET(l4_hdr, skb) ((u32)((l4_hdr) - (skb)->data))
|
|
|
+
|
|
|
+union hinic_l3 {
|
|
|
+ struct iphdr *v4;
|
|
|
+ struct ipv6hdr *v6;
|
|
|
+ unsigned char *hdr;
|
|
|
+};
|
|
|
+
|
|
|
+union hinic_l4 {
|
|
|
+ struct tcphdr *tcp;
|
|
|
+ struct udphdr *udp;
|
|
|
+ unsigned char *hdr;
|
|
|
+};
|
|
|
+
|
|
|
+enum hinic_offload_type {
|
|
|
+ TX_OFFLOAD_TSO = BIT(0),
|
|
|
+ TX_OFFLOAD_CSUM = BIT(1),
|
|
|
+ TX_OFFLOAD_VLAN = BIT(2),
|
|
|
+ TX_OFFLOAD_INVALID = BIT(3),
|
|
|
+};
|
|
|
|
|
|
/**
|
|
|
* hinic_txq_clean_stats - Clean the statistics of specific queue
|
|
@@ -175,18 +204,263 @@ static void tx_unmap_skb(struct hinic_dev *nic_dev, struct sk_buff *skb,
|
|
|
DMA_TO_DEVICE);
|
|
|
}
|
|
|
|
|
|
+static void get_inner_l3_l4_type(struct sk_buff *skb, union hinic_l3 *ip,
|
|
|
+ union hinic_l4 *l4,
|
|
|
+ enum hinic_offload_type offload_type,
|
|
|
+ enum hinic_l3_offload_type *l3_type,
|
|
|
+ u8 *l4_proto)
|
|
|
+{
|
|
|
+ u8 *exthdr;
|
|
|
+
|
|
|
+ if (ip->v4->version == 4) {
|
|
|
+ *l3_type = (offload_type == TX_OFFLOAD_CSUM) ?
|
|
|
+ IPV4_PKT_NO_CHKSUM_OFFLOAD :
|
|
|
+ IPV4_PKT_WITH_CHKSUM_OFFLOAD;
|
|
|
+ *l4_proto = ip->v4->protocol;
|
|
|
+ } else if (ip->v4->version == 6) {
|
|
|
+ *l3_type = IPV6_PKT;
|
|
|
+ exthdr = ip->hdr + sizeof(*ip->v6);
|
|
|
+ *l4_proto = ip->v6->nexthdr;
|
|
|
+ if (exthdr != l4->hdr) {
|
|
|
+ int start = exthdr - skb->data;
|
|
|
+ __be16 frag_off;
|
|
|
+
|
|
|
+ ipv6_skip_exthdr(skb, start, l4_proto, &frag_off);
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ *l3_type = L3TYPE_UNKNOWN;
|
|
|
+ *l4_proto = 0;
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+static void get_inner_l4_info(struct sk_buff *skb, union hinic_l4 *l4,
|
|
|
+ enum hinic_offload_type offload_type, u8 l4_proto,
|
|
|
+ enum hinic_l4_offload_type *l4_offload,
|
|
|
+ u32 *l4_len, u32 *offset)
|
|
|
+{
|
|
|
+ *l4_offload = OFFLOAD_DISABLE;
|
|
|
+ *offset = 0;
|
|
|
+ *l4_len = 0;
|
|
|
+
|
|
|
+ switch (l4_proto) {
|
|
|
+ case IPPROTO_TCP:
|
|
|
+ *l4_offload = TCP_OFFLOAD_ENABLE;
|
|
|
+ /* doff in unit of 4B */
|
|
|
+ *l4_len = l4->tcp->doff * 4;
|
|
|
+ *offset = *l4_len + TRANSPORT_OFFSET(l4->hdr, skb);
|
|
|
+ break;
|
|
|
+
|
|
|
+ case IPPROTO_UDP:
|
|
|
+ *l4_offload = UDP_OFFLOAD_ENABLE;
|
|
|
+ *l4_len = sizeof(struct udphdr);
|
|
|
+ *offset = TRANSPORT_OFFSET(l4->hdr, skb);
|
|
|
+ break;
|
|
|
+
|
|
|
+ case IPPROTO_SCTP:
|
|
|
+ /* only csum offload support sctp */
|
|
|
+ if (offload_type != TX_OFFLOAD_CSUM)
|
|
|
+ break;
|
|
|
+
|
|
|
+ *l4_offload = SCTP_OFFLOAD_ENABLE;
|
|
|
+ *l4_len = sizeof(struct sctphdr);
|
|
|
+ *offset = TRANSPORT_OFFSET(l4->hdr, skb);
|
|
|
+ break;
|
|
|
+
|
|
|
+ default:
|
|
|
+ break;
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+static __sum16 csum_magic(union hinic_l3 *ip, unsigned short proto)
|
|
|
+{
|
|
|
+ return (ip->v4->version == 4) ?
|
|
|
+ csum_tcpudp_magic(ip->v4->saddr, ip->v4->daddr, 0, proto, 0) :
|
|
|
+ csum_ipv6_magic(&ip->v6->saddr, &ip->v6->daddr, 0, proto, 0);
|
|
|
+}
|
|
|
+
|
|
|
+static int offload_tso(struct hinic_sq_task *task, u32 *queue_info,
|
|
|
+ struct sk_buff *skb)
|
|
|
+{
|
|
|
+ u32 offset, l4_len, ip_identify, network_hdr_len;
|
|
|
+ enum hinic_l3_offload_type l3_offload;
|
|
|
+ enum hinic_l4_offload_type l4_offload;
|
|
|
+ union hinic_l3 ip;
|
|
|
+ union hinic_l4 l4;
|
|
|
+ u8 l4_proto;
|
|
|
+
|
|
|
+ if (!skb_is_gso(skb))
|
|
|
+ return 0;
|
|
|
+
|
|
|
+ if (skb_cow_head(skb, 0) < 0)
|
|
|
+ return -EPROTONOSUPPORT;
|
|
|
+
|
|
|
+ if (skb->encapsulation) {
|
|
|
+ u32 gso_type = skb_shinfo(skb)->gso_type;
|
|
|
+ u32 tunnel_type = 0;
|
|
|
+ u32 l4_tunnel_len;
|
|
|
+
|
|
|
+ ip.hdr = skb_network_header(skb);
|
|
|
+ l4.hdr = skb_transport_header(skb);
|
|
|
+ network_hdr_len = skb_inner_network_header_len(skb);
|
|
|
+
|
|
|
+ if (ip.v4->version == 4) {
|
|
|
+ ip.v4->tot_len = 0;
|
|
|
+ l3_offload = IPV4_PKT_WITH_CHKSUM_OFFLOAD;
|
|
|
+ } else if (ip.v4->version == 6) {
|
|
|
+ l3_offload = IPV6_PKT;
|
|
|
+ } else {
|
|
|
+ l3_offload = 0;
|
|
|
+ }
|
|
|
+
|
|
|
+ hinic_task_set_outter_l3(task, l3_offload,
|
|
|
+ skb_network_header_len(skb));
|
|
|
+
|
|
|
+ if (gso_type & SKB_GSO_UDP_TUNNEL_CSUM) {
|
|
|
+ l4.udp->check = ~csum_magic(&ip, IPPROTO_UDP);
|
|
|
+ tunnel_type = TUNNEL_UDP_CSUM;
|
|
|
+ } else if (gso_type & SKB_GSO_UDP_TUNNEL) {
|
|
|
+ tunnel_type = TUNNEL_UDP_NO_CSUM;
|
|
|
+ }
|
|
|
+
|
|
|
+ l4_tunnel_len = skb_inner_network_offset(skb) -
|
|
|
+ skb_transport_offset(skb);
|
|
|
+ hinic_task_set_tunnel_l4(task, tunnel_type, l4_tunnel_len);
|
|
|
+
|
|
|
+ ip.hdr = skb_inner_network_header(skb);
|
|
|
+ l4.hdr = skb_inner_transport_header(skb);
|
|
|
+ } else {
|
|
|
+ ip.hdr = skb_network_header(skb);
|
|
|
+ l4.hdr = skb_transport_header(skb);
|
|
|
+ network_hdr_len = skb_network_header_len(skb);
|
|
|
+ }
|
|
|
+
|
|
|
+ /* initialize inner IP header fields */
|
|
|
+ if (ip.v4->version == 4)
|
|
|
+ ip.v4->tot_len = 0;
|
|
|
+ else
|
|
|
+ ip.v6->payload_len = 0;
|
|
|
+
|
|
|
+ get_inner_l3_l4_type(skb, &ip, &l4, TX_OFFLOAD_TSO, &l3_offload,
|
|
|
+ &l4_proto);
|
|
|
+
|
|
|
+ hinic_task_set_inner_l3(task, l3_offload, network_hdr_len);
|
|
|
+
|
|
|
+ ip_identify = 0;
|
|
|
+ if (l4_proto == IPPROTO_TCP)
|
|
|
+ l4.tcp->check = ~csum_magic(&ip, IPPROTO_TCP);
|
|
|
+
|
|
|
+ get_inner_l4_info(skb, &l4, TX_OFFLOAD_TSO, l4_proto, &l4_offload,
|
|
|
+ &l4_len, &offset);
|
|
|
+
|
|
|
+ hinic_set_tso_inner_l4(task, queue_info, l4_offload, l4_len, offset,
|
|
|
+ ip_identify, skb_shinfo(skb)->gso_size);
|
|
|
+
|
|
|
+ return 1;
|
|
|
+}
|
|
|
+
|
|
|
+static int offload_csum(struct hinic_sq_task *task, u32 *queue_info,
|
|
|
+ struct sk_buff *skb)
|
|
|
+{
|
|
|
+ enum hinic_l4_offload_type l4_offload;
|
|
|
+ u32 offset, l4_len, network_hdr_len;
|
|
|
+ enum hinic_l3_offload_type l3_type;
|
|
|
+ union hinic_l3 ip;
|
|
|
+ union hinic_l4 l4;
|
|
|
+ u8 l4_proto;
|
|
|
+
|
|
|
+ if (skb->ip_summed != CHECKSUM_PARTIAL)
|
|
|
+ return 0;
|
|
|
+
|
|
|
+ if (skb->encapsulation) {
|
|
|
+ u32 l4_tunnel_len;
|
|
|
+
|
|
|
+ ip.hdr = skb_network_header(skb);
|
|
|
+
|
|
|
+ if (ip.v4->version == 4)
|
|
|
+ l3_type = IPV4_PKT_NO_CHKSUM_OFFLOAD;
|
|
|
+ else if (ip.v4->version == 6)
|
|
|
+ l3_type = IPV6_PKT;
|
|
|
+ else
|
|
|
+ l3_type = L3TYPE_UNKNOWN;
|
|
|
+
|
|
|
+ hinic_task_set_outter_l3(task, l3_type,
|
|
|
+ skb_network_header_len(skb));
|
|
|
+
|
|
|
+ l4_tunnel_len = skb_inner_network_offset(skb) -
|
|
|
+ skb_transport_offset(skb);
|
|
|
+
|
|
|
+ hinic_task_set_tunnel_l4(task, TUNNEL_UDP_NO_CSUM,
|
|
|
+ l4_tunnel_len);
|
|
|
+
|
|
|
+ ip.hdr = skb_inner_network_header(skb);
|
|
|
+ l4.hdr = skb_inner_transport_header(skb);
|
|
|
+ network_hdr_len = skb_inner_network_header_len(skb);
|
|
|
+ } else {
|
|
|
+ ip.hdr = skb_network_header(skb);
|
|
|
+ l4.hdr = skb_transport_header(skb);
|
|
|
+ network_hdr_len = skb_network_header_len(skb);
|
|
|
+ }
|
|
|
+
|
|
|
+ get_inner_l3_l4_type(skb, &ip, &l4, TX_OFFLOAD_CSUM, &l3_type,
|
|
|
+ &l4_proto);
|
|
|
+
|
|
|
+ hinic_task_set_inner_l3(task, l3_type, network_hdr_len);
|
|
|
+
|
|
|
+ get_inner_l4_info(skb, &l4, TX_OFFLOAD_CSUM, l4_proto, &l4_offload,
|
|
|
+ &l4_len, &offset);
|
|
|
+
|
|
|
+ hinic_set_cs_inner_l4(task, queue_info, l4_offload, l4_len, offset);
|
|
|
+
|
|
|
+ return 1;
|
|
|
+}
|
|
|
+
|
|
|
+static int hinic_tx_offload(struct sk_buff *skb, struct hinic_sq_task *task,
|
|
|
+ u32 *queue_info)
|
|
|
+{
|
|
|
+ enum hinic_offload_type offload = 0;
|
|
|
+ int enabled;
|
|
|
+
|
|
|
+ enabled = offload_tso(task, queue_info, skb);
|
|
|
+ if (enabled > 0) {
|
|
|
+ offload |= TX_OFFLOAD_TSO;
|
|
|
+ } else if (enabled == 0) {
|
|
|
+ enabled = offload_csum(task, queue_info, skb);
|
|
|
+ if (enabled)
|
|
|
+ offload |= TX_OFFLOAD_CSUM;
|
|
|
+ } else {
|
|
|
+ return -EPROTONOSUPPORT;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (offload)
|
|
|
+ hinic_task_set_l2hdr(task, skb_network_offset(skb));
|
|
|
+
|
|
|
+ /* payload offset should not more than 221 */
|
|
|
+ if (HINIC_SQ_CTRL_GET(*queue_info, QUEUE_INFO_PLDOFF) >
|
|
|
+ MAX_PAYLOAD_OFFSET) {
|
|
|
+ return -EPROTONOSUPPORT;
|
|
|
+ }
|
|
|
+
|
|
|
+ /* mss should not less than 80 */
|
|
|
+ if (HINIC_SQ_CTRL_GET(*queue_info, QUEUE_INFO_MSS) < HINIC_MSS_MIN) {
|
|
|
+ *queue_info = HINIC_SQ_CTRL_CLEAR(*queue_info, QUEUE_INFO_MSS);
|
|
|
+ *queue_info |= HINIC_SQ_CTRL_SET(HINIC_MSS_MIN, QUEUE_INFO_MSS);
|
|
|
+ }
|
|
|
+
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
netdev_tx_t hinic_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
|
|
|
{
|
|
|
struct hinic_dev *nic_dev = netdev_priv(netdev);
|
|
|
+ u16 prod_idx, q_id = skb->queue_mapping;
|
|
|
struct netdev_queue *netdev_txq;
|
|
|
int nr_sges, err = NETDEV_TX_OK;
|
|
|
struct hinic_sq_wqe *sq_wqe;
|
|
|
unsigned int wqe_size;
|
|
|
struct hinic_txq *txq;
|
|
|
struct hinic_qp *qp;
|
|
|
- u16 prod_idx;
|
|
|
|
|
|
- txq = &nic_dev->txqs[skb->queue_mapping];
|
|
|
+ txq = &nic_dev->txqs[q_id];
|
|
|
qp = container_of(txq->sq, struct hinic_qp, sq);
|
|
|
|
|
|
if (skb->len < MIN_SKB_LEN) {
|
|
@@ -236,15 +510,23 @@ netdev_tx_t hinic_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
|
|
|
process_sq_wqe:
|
|
|
hinic_sq_prepare_wqe(txq->sq, prod_idx, sq_wqe, txq->sges, nr_sges);
|
|
|
|
|
|
+ err = hinic_tx_offload(skb, &sq_wqe->task, &sq_wqe->ctrl.queue_info);
|
|
|
+ if (err)
|
|
|
+ goto offload_error;
|
|
|
+
|
|
|
hinic_sq_write_wqe(txq->sq, prod_idx, sq_wqe, skb, wqe_size);
|
|
|
|
|
|
flush_skbs:
|
|
|
- netdev_txq = netdev_get_tx_queue(netdev, skb->queue_mapping);
|
|
|
+ netdev_txq = netdev_get_tx_queue(netdev, q_id);
|
|
|
if ((!skb->xmit_more) || (netif_xmit_stopped(netdev_txq)))
|
|
|
hinic_sq_write_db(txq->sq, prod_idx, wqe_size, 0);
|
|
|
|
|
|
return err;
|
|
|
|
|
|
+offload_error:
|
|
|
+ hinic_sq_return_wqe(txq->sq, wqe_size);
|
|
|
+ tx_unmap_skb(nic_dev, skb, txq->sges);
|
|
|
+
|
|
|
skb_error:
|
|
|
dev_kfree_skb_any(skb);
|
|
|
|
|
@@ -252,7 +534,8 @@ update_error_stats:
|
|
|
u64_stats_update_begin(&txq->txq_stats.syncp);
|
|
|
txq->txq_stats.tx_dropped++;
|
|
|
u64_stats_update_end(&txq->txq_stats.syncp);
|
|
|
- return err;
|
|
|
+
|
|
|
+ return NETDEV_TX_OK;
|
|
|
}
|
|
|
|
|
|
/**
|