diff options
Diffstat (limited to 'drivers/net/ethernet/chelsio/cxgb4/l2t.c')
| -rw-r--r-- | drivers/net/ethernet/chelsio/cxgb4/l2t.c | 357 |
1 files changed, 236 insertions, 121 deletions
diff --git a/drivers/net/ethernet/chelsio/cxgb4/l2t.c b/drivers/net/ethernet/chelsio/cxgb4/l2t.c index 29878098101e..c02b4e9c06b2 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/l2t.c +++ b/drivers/net/ethernet/chelsio/cxgb4/l2t.c @@ -1,7 +1,7 @@ /* * This file is part of the Chelsio T4 Ethernet driver for Linux. * - * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved. + * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -45,33 +45,26 @@ #include "l2t.h" #include "t4_msg.h" #include "t4fw_api.h" - -#define VLAN_NONE 0xfff +#include "t4_regs.h" +#include "t4_values.h" /* identifies sync vs async L2T_WRITE_REQs */ -#define F_SYNC_WR (1 << 12) - -enum { - L2T_STATE_VALID, /* entry is up to date */ - L2T_STATE_STALE, /* entry may be used but needs revalidation */ - L2T_STATE_RESOLVING, /* entry needs address resolution */ - L2T_STATE_SYNC_WRITE, /* synchronous write of entry underway */ - - /* when state is one of the below the entry is not hashed */ - L2T_STATE_SWITCHING, /* entry is being used by a switching filter */ - L2T_STATE_UNUSED /* entry not in use */ -}; +#define SYNC_WR_S 12 +#define SYNC_WR_V(x) ((x) << SYNC_WR_S) +#define SYNC_WR_F SYNC_WR_V(1) struct l2t_data { + unsigned int l2t_start; /* start index of our piece of the L2T */ + unsigned int l2t_size; /* number of entries in l2tab */ rwlock_t lock; atomic_t nfree; /* number of free entries */ struct l2t_entry *rover; /* starting point for next allocation */ - struct l2t_entry l2tab[L2T_SIZE]; + struct l2t_entry l2tab[] __counted_by(l2t_size); /* MUST BE LAST */ }; static inline unsigned int vlan_prio(const struct l2t_entry *e) { - return e->vlan >> 13; + return e->vlan >> VLAN_PRIO_SHIFT; } static inline void l2t_hold(struct l2t_data *d, struct l2t_entry *e) @@ -83,29 +76,36 @@ static inline void l2t_hold(struct l2t_data *d, struct l2t_entry *e) /* * To avoid having to check address families we do not allow v4 and v6 * neighbors to be on the same hash chain. We keep v4 entries in the first - * half of available hash buckets and v6 in the second. + * half of available hash buckets and v6 in the second. We need at least two + * entries in our L2T for this scheme to work. */ enum { - L2T_SZ_HALF = L2T_SIZE / 2, - L2T_HASH_MASK = L2T_SZ_HALF - 1 + L2T_MIN_HASH_BUCKETS = 2, }; -static inline unsigned int arp_hash(const u32 *key, int ifindex) +static inline unsigned int arp_hash(struct l2t_data *d, const u32 *key, + int ifindex) { - return jhash_2words(*key, ifindex, 0) & L2T_HASH_MASK; + unsigned int l2t_size_half = d->l2t_size / 2; + + return jhash_2words(*key, ifindex, 0) % l2t_size_half; } -static inline unsigned int ipv6_hash(const u32 *key, int ifindex) +static inline unsigned int ipv6_hash(struct l2t_data *d, const u32 *key, + int ifindex) { + unsigned int l2t_size_half = d->l2t_size / 2; u32 xor = key[0] ^ key[1] ^ key[2] ^ key[3]; - return L2T_SZ_HALF + (jhash_2words(xor, ifindex, 0) & L2T_HASH_MASK); + return (l2t_size_half + + (jhash_2words(xor, ifindex, 0) % l2t_size_half)); } -static unsigned int addr_hash(const u32 *addr, int addr_len, int ifindex) +static unsigned int addr_hash(struct l2t_data *d, const u32 *addr, + int addr_len, int ifindex) { - return addr_len == 4 ? arp_hash(addr, ifindex) : - ipv6_hash(addr, ifindex); + return addr_len == 4 ? arp_hash(d, addr, ifindex) : + ipv6_hash(d, addr, ifindex); } /* @@ -137,6 +137,8 @@ static void neigh_replace(struct l2t_entry *e, struct neighbour *n) */ static int write_l2e(struct adapter *adap, struct l2t_entry *e, int sync) { + struct l2t_data *d = adap->l2t; + unsigned int l2t_idx = e->idx + d->l2t_start; struct sk_buff *skb; struct cpl_l2t_write_req *req; @@ -144,21 +146,20 @@ static int write_l2e(struct adapter *adap, struct l2t_entry *e, int sync) if (!skb) return -ENOMEM; - req = (struct cpl_l2t_write_req *)__skb_put(skb, sizeof(*req)); + req = __skb_put(skb, sizeof(*req)); INIT_TP_WR(req, 0); OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, - e->idx | (sync ? F_SYNC_WR : 0) | - TID_QID(adap->sge.fw_evtq.abs_id))); - req->params = htons(L2T_W_PORT(e->lport) | L2T_W_NOREPLY(!sync)); - req->l2t_idx = htons(e->idx); + l2t_idx | (sync ? SYNC_WR_F : 0) | + TID_QID_V(adap->sge.fw_evtq.abs_id))); + req->params = htons(L2T_W_PORT_V(e->lport) | L2T_W_NOREPLY_V(!sync)); + req->l2t_idx = htons(l2t_idx); req->vlan = htons(e->vlan); - if (e->neigh) + if (e->neigh && !(e->neigh->dev->flags & IFF_LOOPBACK)) memcpy(e->dmac, e->neigh->ha, sizeof(e->dmac)); memcpy(req->dst_mac, e->dmac, sizeof(req->dst_mac)); - set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); - t4_ofld_send(adap, skb); + t4_mgmt_tx(adap, skb); if (sync && e->state != L2T_STATE_SWITCHING) e->state = L2T_STATE_SYNC_WRITE; @@ -171,14 +172,10 @@ static int write_l2e(struct adapter *adap, struct l2t_entry *e, int sync) */ static void send_pending(struct adapter *adap, struct l2t_entry *e) { - while (e->arpq_head) { - struct sk_buff *skb = e->arpq_head; + struct sk_buff *skb; - e->arpq_head = skb->next; - skb->next = NULL; + while ((skb = __skb_dequeue(&e->arpq)) != NULL) t4_ofld_send(adap, skb); - } - e->arpq_tail = NULL; } /* @@ -188,18 +185,19 @@ static void send_pending(struct adapter *adap, struct l2t_entry *e) */ void do_l2t_write_rpl(struct adapter *adap, const struct cpl_l2t_write_rpl *rpl) { + struct l2t_data *d = adap->l2t; unsigned int tid = GET_TID(rpl); - unsigned int idx = tid & (L2T_SIZE - 1); + unsigned int l2t_idx = tid % L2T_SIZE; if (unlikely(rpl->status != CPL_ERR_NONE)) { dev_err(adap->pdev_dev, "Unexpected L2T_WRITE_RPL status %u for entry %u\n", - rpl->status, idx); + rpl->status, l2t_idx); return; } - if (tid & F_SYNC_WR) { - struct l2t_entry *e = &adap->l2t->l2tab[idx]; + if (tid & SYNC_WR_F) { + struct l2t_entry *e = &d->l2tab[l2t_idx - d->l2t_start]; spin_lock(&e->lock); if (e->state != L2T_STATE_SWITCHING) { @@ -217,12 +215,7 @@ void do_l2t_write_rpl(struct adapter *adap, const struct cpl_l2t_write_rpl *rpl) */ static inline void arpq_enqueue(struct l2t_entry *e, struct sk_buff *skb) { - skb->next = NULL; - if (e->arpq_head) - e->arpq_tail->next = skb; - else - e->arpq_head = skb; - e->arpq_tail = skb; + __skb_queue_tail(&e->arpq, skb); } int cxgb4_l2t_send(struct net_device *dev, struct sk_buff *skb, @@ -238,6 +231,7 @@ again: if (e->state == L2T_STATE_STALE) e->state = L2T_STATE_VALID; spin_unlock_bh(&e->lock); + fallthrough; case L2T_STATE_VALID: /* fast-path, send the packet on */ return t4_ofld_send(adap, skb); case L2T_STATE_RESOLVING: @@ -254,7 +248,8 @@ again: if (e->state == L2T_STATE_RESOLVING && !neigh_event_send(e->neigh, NULL)) { spin_lock_bh(&e->lock); - if (e->state == L2T_STATE_RESOLVING && e->arpq_head) + if (e->state == L2T_STATE_RESOLVING && + !skb_queue_empty(&e->arpq)) write_l2e(adap, e, 1); spin_unlock_bh(&e->lock); } @@ -274,7 +269,7 @@ static struct l2t_entry *alloc_l2e(struct l2t_data *d) return NULL; /* there's definitely a free entry */ - for (e = d->rover, end = &d->l2tab[L2T_SIZE]; e != end; ++e) + for (e = d->rover, end = &d->l2tab[d->l2t_size]; e != end; ++e) if (atomic_read(&e->refcnt) == 0) goto found; @@ -300,26 +295,87 @@ found: return e; } -/* - * Called when an L2T entry has no more users. +static struct l2t_entry *find_or_alloc_l2e(struct l2t_data *d, u16 vlan, + u8 port, u8 *dmac) +{ + struct l2t_entry *end, *e, **p; + struct l2t_entry *first_free = NULL; + + for (e = &d->l2tab[0], end = &d->l2tab[d->l2t_size]; e != end; ++e) { + if (atomic_read(&e->refcnt) == 0) { + if (!first_free) + first_free = e; + } else { + if (e->state == L2T_STATE_SWITCHING) { + if (ether_addr_equal(e->dmac, dmac) && + (e->vlan == vlan) && (e->lport == port)) + goto exists; + } + } + } + + if (first_free) { + e = first_free; + goto found; + } + + return NULL; + +found: + /* The entry we found may be an inactive entry that is + * presently in the hash table. We need to remove it. + */ + if (e->state < L2T_STATE_SWITCHING) + for (p = &d->l2tab[e->hash].first; *p; p = &(*p)->next) + if (*p == e) { + *p = e->next; + e->next = NULL; + break; + } + e->state = L2T_STATE_UNUSED; + +exists: + return e; +} + +/* Called when an L2T entry has no more users. The entry is left in the hash + * table since it is likely to be reused but we also bump nfree to indicate + * that the entry can be reallocated for a different neighbor. We also drop + * the existing neighbor reference in case the neighbor is going away and is + * waiting on our reference. + * + * Because entries can be reallocated to other neighbors once their ref count + * drops to 0 we need to take the entry's lock to avoid races with a new + * incarnation. */ -static void t4_l2e_free(struct l2t_entry *e) +static void _t4_l2e_free(struct l2t_entry *e) { struct l2t_data *d; - spin_lock_bh(&e->lock); if (atomic_read(&e->refcnt) == 0) { /* hasn't been recycled */ if (e->neigh) { neigh_release(e->neigh); e->neigh = NULL; } - while (e->arpq_head) { - struct sk_buff *skb = e->arpq_head; + __skb_queue_purge(&e->arpq); + } + + d = container_of(e, struct l2t_data, l2tab[e->idx]); + atomic_inc(&d->nfree); +} - e->arpq_head = skb->next; - kfree_skb(skb); +/* Locked version of _t4_l2e_free */ +static void t4_l2e_free(struct l2t_entry *e) +{ + struct l2t_data *d; + + spin_lock_bh(&e->lock); + if (atomic_read(&e->refcnt) == 0) { /* hasn't been recycled */ + if (e->neigh) { + neigh_release(e->neigh); + e->neigh = NULL; } - e->arpq_tail = NULL; + __skb_queue_purge(&e->arpq); } spin_unlock_bh(&e->lock); @@ -363,20 +419,22 @@ struct l2t_entry *cxgb4_l2t_get(struct l2t_data *d, struct neighbour *neigh, u8 lport; u16 vlan; struct l2t_entry *e; - int addr_len = neigh->tbl->key_len; + unsigned int addr_len = neigh->tbl->key_len; u32 *addr = (u32 *)neigh->primary_key; int ifidx = neigh->dev->ifindex; - int hash = addr_hash(addr, addr_len, ifidx); + int hash = addr_hash(d, addr, addr_len, ifidx); if (neigh->dev->flags & IFF_LOOPBACK) lport = netdev2pinfo(physdev)->tx_chan + 4; else lport = netdev2pinfo(physdev)->lport; - if (neigh->dev->priv_flags & IFF_802_1Q_VLAN) + if (is_vlan_dev(neigh->dev)) { vlan = vlan_dev_vlan_id(neigh->dev); - else + vlan |= vlan_dev_get_egress_qos_mask(neigh->dev, priority); + } else { vlan = VLAN_NONE; + } write_lock_bh(&d->lock); for (e = d->l2tab[hash].first; e; e = e->next) @@ -393,6 +451,8 @@ struct l2t_entry *cxgb4_l2t_get(struct l2t_data *d, struct neighbour *neigh, if (e) { spin_lock(&e->lock); /* avoid race with t4_l2t_free */ e->state = L2T_STATE_RESOLVING; + if (neigh->dev->flags & IFF_LOOPBACK) + memcpy(e->dmac, physdev->dev_addr, sizeof(e->dmac)); memcpy(e->addr, addr, addr_len); e->ifindex = ifidx; e->hash = hash; @@ -411,25 +471,36 @@ done: } EXPORT_SYMBOL(cxgb4_l2t_get); -/* - * Called when address resolution fails for an L2T entry to handle packets - * on the arpq head. If a packet specifies a failure handler it is invoked, - * otherwise the packet is sent to the device. - */ -static void handle_failed_resolution(struct adapter *adap, struct sk_buff *arpq) +u64 cxgb4_select_ntuple(struct net_device *dev, + const struct l2t_entry *l2t) { - while (arpq) { - struct sk_buff *skb = arpq; - const struct l2t_skb_cb *cb = L2T_SKB_CB(skb); - - arpq = skb->next; - skb->next = NULL; - if (cb->arp_err_handler) - cb->arp_err_handler(cb->handle, skb); - else - t4_ofld_send(adap, skb); + struct adapter *adap = netdev2adap(dev); + struct tp_params *tp = &adap->params.tp; + u64 ntuple = 0; + + /* Initialize each of the fields which we care about which are present + * in the Compressed Filter Tuple. + */ + if (tp->vlan_shift >= 0 && l2t->vlan != VLAN_NONE) + ntuple |= (u64)(FT_VLAN_VLD_F | l2t->vlan) << tp->vlan_shift; + + if (tp->port_shift >= 0) + ntuple |= (u64)l2t->lport << tp->port_shift; + + if (tp->protocol_shift >= 0) + ntuple |= (u64)IPPROTO_TCP << tp->protocol_shift; + + if (tp->vnic_shift >= 0 && (tp->ingress_config & VNIC_F)) { + struct port_info *pi = (struct port_info *)netdev_priv(dev); + + ntuple |= (u64)(FT_VNID_ID_VF_V(pi->vin) | + FT_VNID_ID_PF_V(adap->pf) | + FT_VNID_ID_VLD_V(pi->vivld)) << tp->vnic_shift; } + + return ntuple; } +EXPORT_SYMBOL(cxgb4_select_ntuple); /* * Called when the host's neighbor layer makes a change to some entry that is @@ -437,14 +508,14 @@ static void handle_failed_resolution(struct adapter *adap, struct sk_buff *arpq) */ void t4_l2t_update(struct adapter *adap, struct neighbour *neigh) { - struct l2t_entry *e; - struct sk_buff *arpq = NULL; - struct l2t_data *d = adap->l2t; - int addr_len = neigh->tbl->key_len; + unsigned int addr_len = neigh->tbl->key_len; u32 *addr = (u32 *) neigh->primary_key; - int ifidx = neigh->dev->ifindex; - int hash = addr_hash(addr, addr_len, ifidx); + int hash, ifidx = neigh->dev->ifindex; + struct sk_buff_head *arpq = NULL; + struct l2t_data *d = adap->l2t; + struct l2t_entry *e; + hash = addr_hash(d, addr, addr_len, ifidx); read_lock_bh(&d->lock); for (e = d->l2tab[hash].first; e; e = e->next) if (!addreq(e, addr) && e->ifindex == ifidx) { @@ -465,10 +536,9 @@ void t4_l2t_update(struct adapter *adap, struct neighbour *neigh) if (e->state == L2T_STATE_RESOLVING) { if (neigh->nud_state & NUD_FAILED) { - arpq = e->arpq_head; - e->arpq_head = e->arpq_tail = NULL; + arpq = &e->arpq; } else if ((neigh->nud_state & (NUD_CONNECTED | NUD_STALE)) && - e->arpq_head) { + !skb_queue_empty(&e->arpq)) { write_l2e(adap, e, 1); } } else { @@ -478,71 +548,104 @@ void t4_l2t_update(struct adapter *adap, struct neighbour *neigh) write_l2e(adap, e, 0); } - spin_unlock_bh(&e->lock); + if (arpq) { + struct sk_buff *skb; + + /* Called when address resolution fails for an L2T + * entry to handle packets on the arpq head. If a + * packet specifies a failure handler it is invoked, + * otherwise the packet is sent to the device. + */ + while ((skb = __skb_dequeue(&e->arpq)) != NULL) { + const struct l2t_skb_cb *cb = L2T_SKB_CB(skb); - if (arpq) - handle_failed_resolution(adap, arpq); + spin_unlock(&e->lock); + if (cb->arp_err_handler) + cb->arp_err_handler(cb->handle, skb); + else + t4_ofld_send(adap, skb); + spin_lock(&e->lock); + } + } + spin_unlock_bh(&e->lock); } /* Allocate an L2T entry for use by a switching rule. Such need to be * explicitly freed and while busy they are not on any hash chain, so normal * address resolution updates do not see them. */ -struct l2t_entry *t4_l2t_alloc_switching(struct l2t_data *d) +struct l2t_entry *t4_l2t_alloc_switching(struct adapter *adap, u16 vlan, + u8 port, u8 *eth_addr) { + struct l2t_data *d = adap->l2t; struct l2t_entry *e; + int ret; write_lock_bh(&d->lock); - e = alloc_l2e(d); + e = find_or_alloc_l2e(d, vlan, port, eth_addr); if (e) { spin_lock(&e->lock); /* avoid race with t4_l2t_free */ - e->state = L2T_STATE_SWITCHING; - atomic_set(&e->refcnt, 1); + if (!atomic_read(&e->refcnt)) { + e->state = L2T_STATE_SWITCHING; + e->vlan = vlan; + e->lport = port; + ether_addr_copy(e->dmac, eth_addr); + atomic_set(&e->refcnt, 1); + ret = write_l2e(adap, e, 0); + if (ret < 0) { + _t4_l2e_free(e); + spin_unlock(&e->lock); + write_unlock_bh(&d->lock); + return NULL; + } + } else { + atomic_inc(&e->refcnt); + } + spin_unlock(&e->lock); } write_unlock_bh(&d->lock); return e; } -/* Sets/updates the contents of a switching L2T entry that has been allocated - * with an earlier call to @t4_l2t_alloc_switching. - */ -int t4_l2t_set_switching(struct adapter *adap, struct l2t_entry *e, u16 vlan, - u8 port, u8 *eth_addr) -{ - e->vlan = vlan; - e->lport = port; - memcpy(e->dmac, eth_addr, ETH_ALEN); - return write_l2e(adap, e, 0); -} - -struct l2t_data *t4_init_l2t(void) +struct l2t_data *t4_init_l2t(unsigned int l2t_start, unsigned int l2t_end) { + unsigned int l2t_size; int i; struct l2t_data *d; - d = t4_alloc_mem(sizeof(*d)); + if (l2t_start >= l2t_end || l2t_end >= L2T_SIZE) + return NULL; + l2t_size = l2t_end - l2t_start + 1; + if (l2t_size < L2T_MIN_HASH_BUCKETS) + return NULL; + + d = kvzalloc(struct_size(d, l2tab, l2t_size), GFP_KERNEL); if (!d) return NULL; + d->l2t_start = l2t_start; + d->l2t_size = l2t_size; + d->rover = d->l2tab; - atomic_set(&d->nfree, L2T_SIZE); + atomic_set(&d->nfree, l2t_size); rwlock_init(&d->lock); - for (i = 0; i < L2T_SIZE; ++i) { + for (i = 0; i < d->l2t_size; ++i) { d->l2tab[i].idx = i; d->l2tab[i].state = L2T_STATE_UNUSED; spin_lock_init(&d->l2tab[i].lock); atomic_set(&d->l2tab[i].refcnt, 0); + skb_queue_head_init(&d->l2tab[i].arpq); } return d; } static inline void *l2t_get_idx(struct seq_file *seq, loff_t pos) { - struct l2t_entry *l2tab = seq->private; + struct l2t_data *d = seq->private; - return pos >= L2T_SIZE ? NULL : &l2tab[pos]; + return pos >= d->l2t_size ? NULL : &d->l2tab[pos]; } static void *l2t_seq_start(struct seq_file *seq, loff_t *pos) @@ -553,8 +656,7 @@ static void *l2t_seq_start(struct seq_file *seq, loff_t *pos) static void *l2t_seq_next(struct seq_file *seq, void *v, loff_t *pos) { v = l2t_get_idx(seq, *pos); - if (v) - ++*pos; + ++(*pos); return v; } @@ -568,13 +670,25 @@ static char l2e_state(const struct l2t_entry *e) case L2T_STATE_VALID: return 'V'; case L2T_STATE_STALE: return 'S'; case L2T_STATE_SYNC_WRITE: return 'W'; - case L2T_STATE_RESOLVING: return e->arpq_head ? 'A' : 'R'; + case L2T_STATE_RESOLVING: + return skb_queue_empty(&e->arpq) ? 'R' : 'A'; case L2T_STATE_SWITCHING: return 'X'; default: return 'U'; } } +bool cxgb4_check_l2t_valid(struct l2t_entry *e) +{ + bool valid; + + spin_lock(&e->lock); + valid = (e->state == L2T_STATE_VALID); + spin_unlock(&e->lock); + return valid; +} +EXPORT_SYMBOL(cxgb4_check_l2t_valid); + static int l2t_seq_show(struct seq_file *seq, void *v) { if (v == SEQ_START_TOKEN) @@ -582,6 +696,7 @@ static int l2t_seq_show(struct seq_file *seq, void *v) "Ethernet address VLAN/P LP State Users Port\n"); else { char ip[60]; + struct l2t_data *d = seq->private; struct l2t_entry *e = v; spin_lock_bh(&e->lock); @@ -590,7 +705,7 @@ static int l2t_seq_show(struct seq_file *seq, void *v) else sprintf(ip, e->v6 ? "%pI6c" : "%pI4", e->addr); seq_printf(seq, "%4u %-25s %17pM %4d %u %2u %c %5u %s\n", - e->idx, ip, e->dmac, + e->idx + d->l2t_start, ip, e->dmac, e->vlan & VLAN_VID_MASK, vlan_prio(e), e->lport, l2e_state(e), atomic_read(&e->refcnt), e->neigh ? e->neigh->dev->name : ""); @@ -614,7 +729,7 @@ static int l2t_seq_open(struct inode *inode, struct file *file) struct adapter *adap = inode->i_private; struct seq_file *seq = file->private_data; - seq->private = adap->l2t->l2tab; + seq->private = adap->l2t; } return rc; } |
