patches/ask/upstream/kernel/0137-net__xfrm__xfrm_state.c.patch
raw ยท 10516 bytes
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index 9e14e45..d685ed7 100644
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -58,6 +58,10 @@ static inline bool xfrm_state_hold_rcu(struct xfrm_state __rcu *x)
return refcount_inc_not_zero(&x->refcnt);
}
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+static unsigned short xfrm_state_handle;
+#endif
+
static inline unsigned int xfrm_dst_hash(struct net *net,
const xfrm_address_t *daddr,
const xfrm_address_t *saddr,
@@ -119,6 +123,9 @@ static void xfrm_hash_transfer(struct hlist_head *list,
struct hlist_head *nsrctable,
struct hlist_head *nspitable,
struct hlist_head *nseqtable,
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ struct hlist_head *nhtable,
+#endif
unsigned int nhashmask)
{
struct hlist_node *tmp;
@@ -150,6 +157,13 @@ static void xfrm_hash_transfer(struct hlist_head *list,
XFRM_STATE_INSERT(byseq, &x->byseq, nseqtable + h,
x->xso.type);
}
+
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ if (x->handle && x->in_byh_hash) {
+ h = x->handle & nhashmask;
+ hlist_add_head_rcu(&x->byh, nhtable + h);
+ }
+#endif
}
}
@@ -162,6 +176,9 @@ static void xfrm_hash_resize(struct work_struct *work)
{
struct net *net = container_of(work, struct net, xfrm.state_hash_work);
struct hlist_head *ndst, *nsrc, *nspi, *nseq, *odst, *osrc, *ospi, *oseq;
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ struct hlist_head *nh, *oh;
+#endif
unsigned long nsize, osize;
unsigned int nhashmask, ohashmask;
int i;
@@ -188,6 +205,16 @@ static void xfrm_hash_resize(struct work_struct *work)
xfrm_hash_free(nspi, nsize);
return;
}
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ nh = xfrm_hash_alloc(nsize);
+ if (!nh) {
+ xfrm_hash_free(ndst, nsize);
+ xfrm_hash_free(nsrc, nsize);
+ xfrm_hash_free(nspi, nsize);
+ xfrm_hash_free(nseq, nsize);
+ return;
+ }
+#endif
spin_lock_bh(&net->xfrm.xfrm_state_lock);
write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
@@ -195,17 +222,27 @@ static void xfrm_hash_resize(struct work_struct *work)
nhashmask = (nsize / sizeof(struct hlist_head)) - 1U;
odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net);
for (i = net->xfrm.state_hmask; i >= 0; i--)
- xfrm_hash_transfer(odst + i, ndst, nsrc, nspi, nseq, nhashmask);
+ xfrm_hash_transfer(odst + i, ndst, nsrc, nspi, nseq,
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ nh,
+#endif
+ nhashmask);
osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net);
ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net);
oseq = xfrm_state_deref_prot(net->xfrm.state_byseq, net);
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ oh = xfrm_state_deref_prot(net->xfrm.state_byh, net);
+#endif
ohashmask = net->xfrm.state_hmask;
rcu_assign_pointer(net->xfrm.state_bydst, ndst);
rcu_assign_pointer(net->xfrm.state_bysrc, nsrc);
rcu_assign_pointer(net->xfrm.state_byspi, nspi);
rcu_assign_pointer(net->xfrm.state_byseq, nseq);
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ rcu_assign_pointer(net->xfrm.state_byh, nh);
+#endif
net->xfrm.state_hmask = nhashmask;
write_seqcount_end(&net->xfrm.xfrm_state_hash_generation);
@@ -219,6 +256,9 @@ static void xfrm_hash_resize(struct work_struct *work)
xfrm_hash_free(osrc, osize);
xfrm_hash_free(ospi, osize);
xfrm_hash_free(oseq, osize);
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ xfrm_hash_free(oh, osize);
+#endif
}
static DEFINE_SPINLOCK(xfrm_state_afinfo_lock);
@@ -744,6 +784,9 @@ struct xfrm_state *xfrm_state_alloc(struct net *net)
INIT_HLIST_NODE(&x->bysrc);
INIT_HLIST_NODE(&x->byspi);
INIT_HLIST_NODE(&x->byseq);
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ INIT_HLIST_NODE(&x->byh);
+#endif
hrtimer_setup(&x->mtimer, xfrm_timer_handler, CLOCK_BOOTTIME,
HRTIMER_MODE_ABS_SOFT);
timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0);
@@ -754,6 +797,12 @@ struct xfrm_state *xfrm_state_alloc(struct net *net)
x->lft.hard_packet_limit = XFRM_INF;
x->replay_maxage = 0;
x->replay_maxdiff = 0;
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ do {
+ x->handle = xfrm_state_handle++;
+ } while (x->handle == 0);
+ x->in_byh_hash = 0;
+#endif
x->pcpu_num = UINT_MAX;
spin_lock_init(&x->lock);
x->mode_data = NULL;
@@ -850,6 +899,12 @@ int __xfrm_state_delete(struct xfrm_state *x)
if (!hlist_unhashed(&x->byspi))
hlist_del_init_rcu(&x->byspi);
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ if (x->handle && x->in_byh_hash) {
+ hlist_del_init_rcu(&x->byh);
+ x->in_byh_hash = 0;
+ }
+#endif
net->xfrm.state_num--;
xfrm_nat_keepalive_state_updated(x);
spin_unlock(&net->xfrm.xfrm_state_lock);
@@ -1582,6 +1637,13 @@ found:
net->xfrm.state_byseq + h,
x->xso.type);
}
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ if (x->handle && !x->in_byh_hash) {
+ h = x->handle & net->xfrm.state_hmask;
+ hlist_add_head_rcu(&x->byh, net->xfrm.state_byh + h);
+ x->in_byh_hash = 1;
+ }
+#endif
x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
hrtimer_start(&x->mtimer,
ktime_set(net->xfrm.sysctl_acq_expires, 0),
@@ -1752,6 +1814,14 @@ static void __xfrm_state_insert(struct xfrm_state *x)
x->xso.type);
}
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ if (x->handle && !x->in_byh_hash) {
+ h = x->handle & net->xfrm.state_hmask;
+ hlist_add_head_rcu(&x->byh, net->xfrm.state_byh + h);
+ x->in_byh_hash = 1;
+ }
+#endif
+
hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT);
if (x->replay_maxage)
mod_timer(&x->rtimer, jiffies + x->replay_maxage);
@@ -1773,6 +1843,9 @@ static void __xfrm_state_bump_genids(struct xfrm_state *xnew)
u32 mark = xnew->mark.v & xnew->mark.m;
u32 if_id = xnew->if_id;
u32 cpu_id = xnew->pcpu_num;
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ u16 parent_sa_handle = 0;
+#endif
h = xfrm_dst_hash(net, &xnew->id.daddr, &xnew->props.saddr, reqid, family);
hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {
@@ -1782,9 +1855,17 @@ static void __xfrm_state_bump_genids(struct xfrm_state *xnew)
x->pcpu_num == cpu_id &&
(mark & x->mark.m) == x->mark.v &&
xfrm_addr_equal(&x->id.daddr, &xnew->id.daddr, family) &&
- xfrm_addr_equal(&x->props.saddr, &xnew->props.saddr, family))
+ xfrm_addr_equal(&x->props.saddr, &xnew->props.saddr, family)) {
x->genid++;
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ if (!parent_sa_handle)
+ parent_sa_handle = x->handle;
+#endif
+ }
}
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ xnew->parent_sa_handle = parent_sa_handle;
+#endif
}
void xfrm_state_insert(struct xfrm_state *x)
@@ -2352,6 +2433,37 @@ xfrm_state_lookup_byaddr(struct net *net, u32 mark,
}
EXPORT_SYMBOL(xfrm_state_lookup_byaddr);
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+struct xfrm_state *__xfrm_state_lookup_byhandle(struct net *net, u16 handle)
+{
+ unsigned int h = handle & net->xfrm.state_hmask;
+ struct xfrm_state *x;
+
+ hlist_for_each_entry(x, net->xfrm.state_byh + h, byh) {
+ if (x->handle != handle)
+ continue;
+
+ xfrm_state_hold(x);
+ return x;
+ }
+
+ return NULL;
+}
+
+struct xfrm_state *
+xfrm_state_lookup_byhandle(struct net *net, u16 handle)
+{
+ struct xfrm_state *x;
+
+ spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ x = __xfrm_state_lookup_byhandle(net, handle);
+ spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+
+ return x;
+}
+EXPORT_SYMBOL(xfrm_state_lookup_byhandle);
+#endif
+
struct xfrm_state *
xfrm_find_acq(struct net *net, const struct xfrm_mark *mark, u8 mode, u32 reqid,
u32 if_id, u32 pcpu_num, u8 proto, const xfrm_address_t *daddr,
@@ -2603,6 +2715,13 @@ int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high,
x->id.spi = newspi;
h = xfrm_spi_hash(net, &x->id.daddr, newspi, x->id.proto, x->props.family);
XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h, x->xso.type);
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ if (x->handle && !x->in_byh_hash) {
+ h = x->handle & net->xfrm.state_hmask;
+ hlist_add_head_rcu(&x->byh, net->xfrm.state_byh + h);
+ x->in_byh_hash = 1;
+ }
+#endif
spin_unlock_bh(&net->xfrm.xfrm_state_lock);
err = 0;
goto unlock;
@@ -3279,6 +3398,12 @@ int __net_init xfrm_state_init(struct net *net)
net->xfrm.state_byseq = xfrm_hash_alloc(sz);
if (!net->xfrm.state_byseq)
goto out_byseq;
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ net->xfrm.state_byh = xfrm_hash_alloc(sz);
+ if (!net->xfrm.state_byh)
+ goto out_byh;
+ get_random_bytes(&xfrm_state_handle, sizeof(xfrm_state_handle));
+#endif
net->xfrm.state_cache_input = alloc_percpu(struct hlist_head);
if (!net->xfrm.state_cache_input)
@@ -3294,6 +3419,10 @@ int __net_init xfrm_state_init(struct net *net)
return 0;
out_state_cache_input:
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ xfrm_hash_free(net->xfrm.state_byh, sz);
+out_byh:
+#endif
xfrm_hash_free(net->xfrm.state_byseq, sz);
out_byseq:
xfrm_hash_free(net->xfrm.state_byspi, sz);
@@ -3321,9 +3450,15 @@ void xfrm_state_fini(struct net *net)
WARN_ON(!hlist_empty(net->xfrm.state_byspi + i));
WARN_ON(!hlist_empty(net->xfrm.state_bysrc + i));
WARN_ON(!hlist_empty(net->xfrm.state_bydst + i));
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ WARN_ON(!hlist_empty(net->xfrm.state_byh + i));
+#endif
}
sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head);
+#if defined(CONFIG_INET_IPSEC_OFFLOAD) || defined(CONFIG_INET6_IPSEC_OFFLOAD)
+ xfrm_hash_free(net->xfrm.state_byh, sz);
+#endif
xfrm_hash_free(net->xfrm.state_byseq, sz);
xfrm_hash_free(net->xfrm.state_byspi, sz);
xfrm_hash_free(net->xfrm.state_bysrc, sz);