600-bridge_offload.patch 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846
  1. From 11c3fae5afa6cac444d12622e2cf5af60a99c1ef Mon Sep 17 00:00:00 2001
  2. From: OpenWrt community <[email protected]>
  3. Date: Wed, 13 Jul 2022 13:43:15 +0200
  4. Subject: [PATCH] net/bridge: add bridge offload
  5. ---
  6. include/linux/if_bridge.h | 1 +
  7. net/bridge/Makefile | 2 +-
  8. net/bridge/br.c | 8 +
  9. net/bridge/br_device.c | 2 +
  10. net/bridge/br_fdb.c | 5 +
  11. net/bridge/br_forward.c | 3 +
  12. net/bridge/br_if.c | 6 +-
  13. net/bridge/br_input.c | 5 +
  14. net/bridge/br_offload.c | 438 ++++++++++++++++++++++++++++++++
  15. net/bridge/br_private.h | 22 +-
  16. net/bridge/br_private_offload.h | 23 ++
  17. net/bridge/br_stp.c | 3 +
  18. net/bridge/br_sysfs_br.c | 35 +++
  19. net/bridge/br_sysfs_if.c | 2 +
  20. net/bridge/br_vlan_tunnel.c | 3 +
  21. 15 files changed, 555 insertions(+), 3 deletions(-)
  22. create mode 100644 net/bridge/br_offload.c
  23. create mode 100644 net/bridge/br_private_offload.h
  24. --- a/include/linux/if_bridge.h
  25. +++ b/include/linux/if_bridge.h
  26. @@ -59,6 +59,7 @@ struct br_ip_list {
  27. #define BR_MRP_LOST_IN_CONT BIT(19)
  28. #define BR_TX_FWD_OFFLOAD BIT(20)
  29. #define BR_BPDU_FILTER BIT(21)
  30. +#define BR_OFFLOAD BIT(22)
  31. #define BR_DEFAULT_AGEING_TIME (300 * HZ)
  32. --- a/net/bridge/Makefile
  33. +++ b/net/bridge/Makefile
  34. @@ -5,7 +5,7 @@
  35. obj-$(CONFIG_BRIDGE) += bridge.o
  36. -bridge-y := br.o br_device.o br_fdb.o br_forward.o br_if.o br_input.o \
  37. +bridge-y := br.o br_device.o br_fdb.o br_forward.o br_if.o br_input.o br_offload.o \
  38. br_ioctl.o br_stp.o br_stp_bpdu.o \
  39. br_stp_if.o br_stp_timer.o br_netlink.o \
  40. br_netlink_tunnel.o br_arp_nd_proxy.o
  41. --- a/net/bridge/br.c
  42. +++ b/net/bridge/br.c
  43. @@ -18,6 +18,7 @@
  44. #include <net/switchdev.h>
  45. #include "br_private.h"
  46. +#include "br_private_offload.h"
  47. /*
  48. * Handle changes in state of network devices enslaved to a bridge.
  49. @@ -381,6 +382,10 @@ static int __init br_init(void)
  50. if (err)
  51. goto err_out;
  52. + err = br_offload_init();
  53. + if (err)
  54. + goto err_out0;
  55. +
  56. err = register_pernet_subsys(&br_net_ops);
  57. if (err)
  58. goto err_out1;
  59. @@ -430,6 +435,8 @@ err_out3:
  60. err_out2:
  61. unregister_pernet_subsys(&br_net_ops);
  62. err_out1:
  63. + br_offload_fini();
  64. +err_out0:
  65. br_fdb_fini();
  66. err_out:
  67. stp_proto_unregister(&br_stp_proto);
  68. @@ -452,6 +459,7 @@ static void __exit br_deinit(void)
  69. #if IS_ENABLED(CONFIG_ATM_LANE)
  70. br_fdb_test_addr_hook = NULL;
  71. #endif
  72. + br_offload_fini();
  73. br_fdb_fini();
  74. }
  75. --- a/net/bridge/br_device.c
  76. +++ b/net/bridge/br_device.c
  77. @@ -524,6 +524,8 @@ void br_dev_setup(struct net_device *dev
  78. br->bridge_hello_time = br->hello_time = 2 * HZ;
  79. br->bridge_forward_delay = br->forward_delay = 15 * HZ;
  80. br->bridge_ageing_time = br->ageing_time = BR_DEFAULT_AGEING_TIME;
  81. + br->offload_cache_size = 128;
  82. + br->offload_cache_reserved = 8;
  83. dev->max_mtu = ETH_MAX_MTU;
  84. br_netfilter_rtable_init(br);
  85. --- a/net/bridge/br_fdb.c
  86. +++ b/net/bridge/br_fdb.c
  87. @@ -23,6 +23,7 @@
  88. #include <net/switchdev.h>
  89. #include <trace/events/bridge.h>
  90. #include "br_private.h"
  91. +#include "br_private_offload.h"
  92. static const struct rhashtable_params br_fdb_rht_params = {
  93. .head_offset = offsetof(struct net_bridge_fdb_entry, rhnode),
  94. @@ -518,6 +519,8 @@ static struct net_bridge_fdb_entry *fdb_
  95. fdb->key.vlan_id = vid;
  96. fdb->flags = flags;
  97. fdb->updated = fdb->used = jiffies;
  98. + INIT_HLIST_HEAD(&fdb->offload_in);
  99. + INIT_HLIST_HEAD(&fdb->offload_out);
  100. if (rhashtable_lookup_insert_fast(&br->fdb_hash_tbl,
  101. &fdb->rhnode,
  102. br_fdb_rht_params)) {
  103. @@ -794,6 +797,8 @@ static void fdb_notify(struct net_bridge
  104. struct sk_buff *skb;
  105. int err = -ENOBUFS;
  106. + br_offload_fdb_update(fdb);
  107. +
  108. if (swdev_notify)
  109. br_switchdev_fdb_notify(br, fdb, type);
  110. --- a/net/bridge/br_forward.c
  111. +++ b/net/bridge/br_forward.c
  112. @@ -16,6 +16,7 @@
  113. #include <linux/if_vlan.h>
  114. #include <linux/netfilter_bridge.h>
  115. #include "br_private.h"
  116. +#include "br_private_offload.h"
  117. /* Don't forward packets to originating port or forwarding disabled */
  118. static inline int should_deliver(const struct net_bridge_port *p,
  119. @@ -32,6 +33,8 @@ static inline int should_deliver(const s
  120. int br_dev_queue_push_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
  121. {
  122. + br_offload_output(skb);
  123. +
  124. skb_push(skb, ETH_HLEN);
  125. if (!is_skb_forwardable(skb->dev, skb))
  126. goto drop;
  127. --- a/net/bridge/br_if.c
  128. +++ b/net/bridge/br_if.c
  129. @@ -25,6 +25,7 @@
  130. #include <net/net_namespace.h>
  131. #include "br_private.h"
  132. +#include "br_private_offload.h"
  133. /*
  134. * Determine initial path cost based on speed.
  135. @@ -428,7 +429,7 @@ static struct net_bridge_port *new_nbp(s
  136. p->path_cost = port_cost(dev);
  137. p->priority = 0x8000 >> BR_PORT_BITS;
  138. p->port_no = index;
  139. - p->flags = BR_LEARNING | BR_FLOOD | BR_MCAST_FLOOD | BR_BCAST_FLOOD;
  140. + p->flags = BR_LEARNING | BR_FLOOD | BR_MCAST_FLOOD | BR_BCAST_FLOOD | BR_OFFLOAD;
  141. br_init_port(p);
  142. br_set_state(p, BR_STATE_DISABLED);
  143. br_stp_port_timer_init(p);
  144. @@ -771,6 +772,9 @@ void br_port_flags_change(struct net_bri
  145. if (mask & BR_NEIGH_SUPPRESS)
  146. br_recalculate_neigh_suppress_enabled(br);
  147. +
  148. + if (mask & BR_OFFLOAD)
  149. + br_offload_port_state(p);
  150. }
  151. bool br_port_flag_is_set(const struct net_device *dev, unsigned long flag)
  152. --- a/net/bridge/br_input.c
  153. +++ b/net/bridge/br_input.c
  154. @@ -22,6 +22,7 @@
  155. #include <linux/rculist.h>
  156. #include "br_private.h"
  157. #include "br_private_tunnel.h"
  158. +#include "br_private_offload.h"
  159. static int
  160. br_netif_receive_skb(struct net *net, struct sock *sk, struct sk_buff *skb)
  161. @@ -171,6 +172,7 @@ int br_handle_frame_finish(struct net *n
  162. dst->used = now;
  163. br_forward(dst->dst, skb, local_rcv, false);
  164. } else {
  165. + br_offload_skb_disable(skb);
  166. if (!mcast_hit)
  167. br_flood(br, skb, pkt_type, local_rcv, false);
  168. else
  169. @@ -304,6 +306,9 @@ static rx_handler_result_t br_handle_fra
  170. memset(skb->cb, 0, sizeof(struct br_input_skb_cb));
  171. p = br_port_get_rcu(skb->dev);
  172. + if (br_offload_input(p, skb))
  173. + return RX_HANDLER_CONSUMED;
  174. +
  175. if (p->flags & BR_VLAN_TUNNEL)
  176. br_handle_ingress_vlan_tunnel(skb, p, nbp_vlan_group_rcu(p));
  177. --- /dev/null
  178. +++ b/net/bridge/br_offload.c
  179. @@ -0,0 +1,438 @@
  180. +// SPDX-License-Identifier: GPL-2.0-only
  181. +#include <linux/kernel.h>
  182. +#include <linux/workqueue.h>
  183. +#include "br_private.h"
  184. +#include "br_private_offload.h"
  185. +
  186. +static DEFINE_SPINLOCK(offload_lock);
  187. +
  188. +struct bridge_flow_key {
  189. + u8 dest[ETH_ALEN];
  190. + u8 src[ETH_ALEN];
  191. +#ifdef CONFIG_BRIDGE_VLAN_FILTERING
  192. + u16 vlan_tag;
  193. + bool vlan_present;
  194. +#endif
  195. +};
  196. +
  197. +struct bridge_flow {
  198. + struct net_bridge_port *port;
  199. + struct rhash_head node;
  200. + struct bridge_flow_key key;
  201. +#ifdef CONFIG_BRIDGE_VLAN_FILTERING
  202. + bool vlan_out_present;
  203. + u16 vlan_out;
  204. +#endif
  205. +
  206. + unsigned long used;
  207. + struct net_bridge_fdb_entry *fdb_in, *fdb_out;
  208. + struct hlist_node fdb_list_in, fdb_list_out;
  209. +
  210. + struct rcu_head rcu;
  211. +};
  212. +
  213. +static const struct rhashtable_params flow_params = {
  214. + .automatic_shrinking = true,
  215. + .head_offset = offsetof(struct bridge_flow, node),
  216. + .key_len = sizeof(struct bridge_flow_key),
  217. + .key_offset = offsetof(struct bridge_flow, key),
  218. +};
  219. +
  220. +static struct kmem_cache *offload_cache __read_mostly;
  221. +
  222. +static void
  223. +flow_rcu_free(struct rcu_head *head)
  224. +{
  225. + struct bridge_flow *flow;
  226. +
  227. + flow = container_of(head, struct bridge_flow, rcu);
  228. + kmem_cache_free(offload_cache, flow);
  229. +}
  230. +
  231. +static void
  232. +__br_offload_flow_free(struct bridge_flow *flow)
  233. +{
  234. + flow->used = 0;
  235. + hlist_del(&flow->fdb_list_in);
  236. + hlist_del(&flow->fdb_list_out);
  237. +
  238. + call_rcu(&flow->rcu, flow_rcu_free);
  239. +}
  240. +
  241. +static void
  242. +br_offload_flow_free(struct bridge_flow *flow)
  243. +{
  244. + if (rhashtable_remove_fast(&flow->port->offload.rht, &flow->node,
  245. + flow_params) != 0)
  246. + return;
  247. +
  248. + __br_offload_flow_free(flow);
  249. +}
  250. +
  251. +static bool
  252. +br_offload_flow_fdb_refresh_time(struct bridge_flow *flow,
  253. + struct net_bridge_fdb_entry *fdb)
  254. +{
  255. + if (!time_after(flow->used, fdb->updated))
  256. + return false;
  257. +
  258. + fdb->updated = flow->used;
  259. +
  260. + return true;
  261. +}
  262. +
  263. +
  264. +static void
  265. +br_offload_flow_refresh_time(struct bridge_flow *flow)
  266. +{
  267. + br_offload_flow_fdb_refresh_time(flow, flow->fdb_in);
  268. + br_offload_flow_fdb_refresh_time(flow, flow->fdb_out);
  269. +}
  270. +
  271. +static void
  272. +br_offload_destroy_cb(void *ptr, void *arg)
  273. +{
  274. + struct bridge_flow *flow = ptr;
  275. +
  276. + __br_offload_flow_free(flow);
  277. +}
  278. +
  279. +static bool
  280. +br_offload_need_gc(struct net_bridge_port *p)
  281. +{
  282. + return (atomic_read(&p->offload.rht.nelems) +
  283. + p->br->offload_cache_reserved) >= p->br->offload_cache_size;
  284. +}
  285. +
  286. +static void
  287. +br_offload_gc_work(struct work_struct *work)
  288. +{
  289. + struct rhashtable_iter hti;
  290. + struct net_bridge_port *p;
  291. + struct bridge_flow *gc_flow = NULL;
  292. + struct bridge_flow *flow;
  293. + unsigned long gc_used;
  294. +
  295. + p = container_of(work, struct net_bridge_port, offload.gc_work);
  296. +
  297. + if (!br_offload_need_gc(p))
  298. + return;
  299. +
  300. + rhashtable_walk_enter(&p->offload.rht, &hti);
  301. + rhashtable_walk_start(&hti);
  302. + while ((flow = rhashtable_walk_next(&hti)) != NULL) {
  303. + unsigned long used;
  304. +
  305. + if (IS_ERR(flow))
  306. + continue;
  307. +
  308. + used = READ_ONCE(flow->used);
  309. + if (!used)
  310. + continue;
  311. +
  312. + if (gc_flow && !time_before(used, gc_used))
  313. + continue;
  314. +
  315. + gc_flow = flow;
  316. + gc_used = used;
  317. + }
  318. + rhashtable_walk_stop(&hti);
  319. + rhashtable_walk_exit(&hti);
  320. +
  321. + if (!gc_flow)
  322. + return;
  323. +
  324. + spin_lock_bh(&offload_lock);
  325. + if (br_offload_need_gc(p) && gc_flow &&
  326. + gc_flow->used == gc_used)
  327. + br_offload_flow_free(gc_flow);
  328. + if (p->offload.enabled && br_offload_need_gc(p))
  329. + queue_work(system_long_wq, work);
  330. + spin_unlock_bh(&offload_lock);
  331. +
  332. +}
  333. +
  334. +void br_offload_port_state(struct net_bridge_port *p)
  335. +{
  336. + struct net_bridge_port_offload *o = &p->offload;
  337. + bool enabled = true;
  338. + bool flush = false;
  339. +
  340. + if (p->state != BR_STATE_FORWARDING ||
  341. + !(p->flags & BR_OFFLOAD))
  342. + enabled = false;
  343. +
  344. + spin_lock_bh(&offload_lock);
  345. + if (o->enabled == enabled)
  346. + goto out;
  347. +
  348. + if (enabled) {
  349. + if (!o->gc_work.func)
  350. + INIT_WORK(&o->gc_work, br_offload_gc_work);
  351. + rhashtable_init(&o->rht, &flow_params);
  352. + } else {
  353. + flush = true;
  354. + rhashtable_free_and_destroy(&o->rht, br_offload_destroy_cb, o);
  355. + }
  356. +
  357. + o->enabled = enabled;
  358. +
  359. +out:
  360. + spin_unlock_bh(&offload_lock);
  361. +
  362. + if (flush)
  363. + flush_work(&o->gc_work);
  364. +}
  365. +
  366. +void br_offload_fdb_update(const struct net_bridge_fdb_entry *fdb)
  367. +{
  368. + struct bridge_flow *f;
  369. + struct hlist_node *tmp;
  370. +
  371. + spin_lock_bh(&offload_lock);
  372. +
  373. + hlist_for_each_entry_safe(f, tmp, &fdb->offload_in, fdb_list_in)
  374. + br_offload_flow_free(f);
  375. +
  376. + hlist_for_each_entry_safe(f, tmp, &fdb->offload_out, fdb_list_out)
  377. + br_offload_flow_free(f);
  378. +
  379. + spin_unlock_bh(&offload_lock);
  380. +}
  381. +
  382. +static void
  383. +br_offload_prepare_key(struct net_bridge_port *p, struct bridge_flow_key *key,
  384. + struct sk_buff *skb)
  385. +{
  386. + memset(key, 0, sizeof(*key));
  387. + memcpy(key, eth_hdr(skb), 2 * ETH_ALEN);
  388. +#ifdef CONFIG_BRIDGE_VLAN_FILTERING
  389. + if (!br_opt_get(p->br, BROPT_VLAN_ENABLED))
  390. + return;
  391. +
  392. + if (!skb_vlan_tag_present(skb) || skb->vlan_proto != p->br->vlan_proto)
  393. + return;
  394. +
  395. + key->vlan_present = true;
  396. + key->vlan_tag = skb_vlan_tag_get_id(skb);
  397. +#endif
  398. +}
  399. +
  400. +void br_offload_output(struct sk_buff *skb)
  401. +{
  402. + struct net_bridge_port_offload *o;
  403. + struct br_input_skb_cb *cb = (struct br_input_skb_cb *)skb->cb;
  404. + struct net_bridge_port *p, *inp;
  405. + struct net_device *dev;
  406. + struct net_bridge_fdb_entry *fdb_in, *fdb_out;
  407. + struct net_bridge_vlan_group *vg;
  408. + struct bridge_flow_key key;
  409. + struct bridge_flow *flow;
  410. + u16 vlan;
  411. +
  412. + if (!cb->offload)
  413. + return;
  414. +
  415. + rcu_read_lock();
  416. +
  417. + p = br_port_get_rcu(skb->dev);
  418. + if (!p)
  419. + goto out;
  420. +
  421. + o = &p->offload;
  422. + if (!o->enabled)
  423. + goto out;
  424. +
  425. + if (atomic_read(&p->offload.rht.nelems) >= p->br->offload_cache_size)
  426. + goto out;
  427. +
  428. + dev = dev_get_by_index_rcu(dev_net(p->br->dev), cb->input_ifindex);
  429. + if (!dev)
  430. + goto out;
  431. +
  432. + inp = br_port_get_rcu(dev);
  433. + if (!inp)
  434. + goto out;
  435. +
  436. + vg = nbp_vlan_group_rcu(inp);
  437. + vlan = cb->input_vlan_present ? cb->input_vlan_tag : br_get_pvid(vg);
  438. + fdb_in = br_fdb_find_rcu(p->br, eth_hdr(skb)->h_source, vlan);
  439. + if (!fdb_in || !fdb_in->dst)
  440. + goto out;
  441. +
  442. + vg = nbp_vlan_group_rcu(p);
  443. + vlan = skb_vlan_tag_present(skb) ? skb_vlan_tag_get_id(skb) : br_get_pvid(vg);
  444. + fdb_out = br_fdb_find_rcu(p->br, eth_hdr(skb)->h_dest, vlan);
  445. + if (!fdb_out || !fdb_out->dst)
  446. + goto out;
  447. +
  448. + br_offload_prepare_key(p, &key, skb);
  449. +#ifdef CONFIG_BRIDGE_VLAN_FILTERING
  450. + key.vlan_present = cb->input_vlan_present;
  451. + key.vlan_tag = cb->input_vlan_tag;
  452. +#endif
  453. +
  454. + flow = kmem_cache_alloc(offload_cache, GFP_ATOMIC);
  455. + flow->port = inp;
  456. + memcpy(&flow->key, &key, sizeof(key));
  457. +
  458. +#ifdef CONFIG_BRIDGE_VLAN_FILTERING
  459. + flow->vlan_out_present = skb_vlan_tag_present(skb);
  460. + flow->vlan_out = skb_vlan_tag_get(skb);
  461. +#endif
  462. +
  463. + flow->fdb_in = fdb_in;
  464. + flow->fdb_out = fdb_out;
  465. + flow->used = jiffies;
  466. +
  467. + spin_lock_bh(&offload_lock);
  468. + if (!o->enabled ||
  469. + atomic_read(&p->offload.rht.nelems) >= p->br->offload_cache_size ||
  470. + rhashtable_insert_fast(&inp->offload.rht, &flow->node, flow_params)) {
  471. + kmem_cache_free(offload_cache, flow);
  472. + goto out_unlock;
  473. + }
  474. +
  475. + hlist_add_head(&flow->fdb_list_in, &fdb_in->offload_in);
  476. + hlist_add_head(&flow->fdb_list_out, &fdb_out->offload_out);
  477. +
  478. + if (br_offload_need_gc(p))
  479. + queue_work(system_long_wq, &p->offload.gc_work);
  480. +
  481. +out_unlock:
  482. + spin_unlock_bh(&offload_lock);
  483. +
  484. +out:
  485. + rcu_read_unlock();
  486. +}
  487. +
  488. +bool br_offload_input(struct net_bridge_port *p, struct sk_buff *skb)
  489. +{
  490. + struct net_bridge_port_offload *o = &p->offload;
  491. + struct br_input_skb_cb *cb = (struct br_input_skb_cb *)skb->cb;
  492. + struct bridge_flow_key key;
  493. + struct net_bridge_port *dst;
  494. + struct bridge_flow *flow;
  495. + unsigned long now = jiffies;
  496. + bool ret = false;
  497. +
  498. + if (skb->len < sizeof(key))
  499. + return false;
  500. +
  501. + if (!o->enabled)
  502. + return false;
  503. +
  504. + if (is_multicast_ether_addr(eth_hdr(skb)->h_dest))
  505. + return false;
  506. +
  507. + br_offload_prepare_key(p, &key, skb);
  508. +
  509. + rcu_read_lock();
  510. + flow = rhashtable_lookup(&o->rht, &key, flow_params);
  511. + if (!flow) {
  512. + cb->offload = 1;
  513. +#ifdef CONFIG_BRIDGE_VLAN_FILTERING
  514. + cb->input_vlan_present = key.vlan_present != 0;
  515. + cb->input_vlan_tag = key.vlan_tag;
  516. +#endif
  517. + cb->input_ifindex = p->dev->ifindex;
  518. + goto out;
  519. + }
  520. +
  521. + if (flow->fdb_in->dst != p)
  522. + goto out;
  523. +
  524. + dst = flow->fdb_out->dst;
  525. + if (!dst)
  526. + goto out;
  527. +
  528. + ret = true;
  529. +#ifdef CONFIG_BRIDGE_VLAN_FILTERING
  530. + if (!flow->vlan_out_present && key.vlan_present) {
  531. + __vlan_hwaccel_clear_tag(skb);
  532. + } else if (flow->vlan_out_present) {
  533. + if (skb_vlan_tag_present(skb) &&
  534. + skb->vlan_proto != p->br->vlan_proto) {
  535. + /* Protocol-mismatch, empty out vlan_tci for new tag */
  536. + skb_push(skb, ETH_HLEN);
  537. + skb = vlan_insert_tag_set_proto(skb, skb->vlan_proto,
  538. + skb_vlan_tag_get(skb));
  539. + if (unlikely(!skb))
  540. + goto out;
  541. +
  542. + skb_pull(skb, ETH_HLEN);
  543. + skb_reset_mac_len(skb);
  544. + }
  545. +
  546. + __vlan_hwaccel_put_tag(skb, p->br->vlan_proto,
  547. + flow->vlan_out);
  548. + }
  549. +#endif
  550. +
  551. + skb->dev = dst->dev;
  552. + skb_push(skb, ETH_HLEN);
  553. +
  554. + if (skb_warn_if_lro(skb) || !is_skb_forwardable(skb->dev, skb)) {
  555. + kfree_skb(skb);
  556. + goto out;
  557. + }
  558. +
  559. + if (now - flow->used >= HZ) {
  560. + flow->used = now;
  561. + br_offload_flow_refresh_time(flow);
  562. + }
  563. +
  564. + skb_forward_csum(skb);
  565. + dev_queue_xmit(skb);
  566. +
  567. +out:
  568. + rcu_read_unlock();
  569. + return ret;
  570. +}
  571. +
  572. +static void
  573. +br_offload_check_gc(struct net_bridge *br)
  574. +{
  575. + struct net_bridge_port *p;
  576. +
  577. + spin_lock_bh(&br->lock);
  578. + list_for_each_entry(p, &br->port_list, list)
  579. + if (br_offload_need_gc(p))
  580. + queue_work(system_long_wq, &p->offload.gc_work);
  581. + spin_unlock_bh(&br->lock);
  582. +}
  583. +
  584. +
  585. +int br_offload_set_cache_size(struct net_bridge *br, unsigned long val,
  586. + struct netlink_ext_ack *extack)
  587. +{
  588. + br->offload_cache_size = val;
  589. + br_offload_check_gc(br);
  590. +
  591. + return 0;
  592. +}
  593. +
  594. +int br_offload_set_cache_reserved(struct net_bridge *br, unsigned long val,
  595. + struct netlink_ext_ack *extack)
  596. +{
  597. + br->offload_cache_reserved = val;
  598. + br_offload_check_gc(br);
  599. +
  600. + return 0;
  601. +}
  602. +
  603. +int __init br_offload_init(void)
  604. +{
  605. + offload_cache = kmem_cache_create("bridge_offload_cache",
  606. + sizeof(struct bridge_flow),
  607. + 0, SLAB_HWCACHE_ALIGN, NULL);
  608. + if (!offload_cache)
  609. + return -ENOMEM;
  610. +
  611. + return 0;
  612. +}
  613. +
  614. +void br_offload_fini(void)
  615. +{
  616. + kmem_cache_destroy(offload_cache);
  617. +}
  618. --- a/net/bridge/br_private.h
  619. +++ b/net/bridge/br_private.h
  620. @@ -268,7 +268,13 @@ struct net_bridge_fdb_entry {
  621. unsigned long updated ____cacheline_aligned_in_smp;
  622. unsigned long used;
  623. - struct rcu_head rcu;
  624. + union {
  625. + struct {
  626. + struct hlist_head offload_in;
  627. + struct hlist_head offload_out;
  628. + };
  629. + struct rcu_head rcu;
  630. + };
  631. };
  632. #define MDB_PG_FLAGS_PERMANENT BIT(0)
  633. @@ -343,6 +349,12 @@ struct net_bridge_mdb_entry {
  634. struct rcu_head rcu;
  635. };
  636. +struct net_bridge_port_offload {
  637. + struct rhashtable rht;
  638. + struct work_struct gc_work;
  639. + bool enabled;
  640. +};
  641. +
  642. struct net_bridge_port {
  643. struct net_bridge *br;
  644. struct net_device *dev;
  645. @@ -403,6 +415,7 @@ struct net_bridge_port {
  646. u16 backup_redirected_cnt;
  647. struct bridge_stp_xstats stp_xstats;
  648. + struct net_bridge_port_offload offload;
  649. };
  650. #define kobj_to_brport(obj) container_of(obj, struct net_bridge_port, kobj)
  651. @@ -519,6 +532,9 @@ struct net_bridge {
  652. struct kobject *ifobj;
  653. u32 auto_cnt;
  654. + u32 offload_cache_size;
  655. + u32 offload_cache_reserved;
  656. +
  657. #ifdef CONFIG_NET_SWITCHDEV
  658. /* Counter used to make sure that hardware domains get unique
  659. * identifiers in case a bridge spans multiple switchdev instances.
  660. @@ -553,6 +569,10 @@ struct br_input_skb_cb {
  661. #ifdef CONFIG_NETFILTER_FAMILY_BRIDGE
  662. u8 br_netfilter_broute:1;
  663. #endif
  664. + u8 offload:1;
  665. + u8 input_vlan_present:1;
  666. + u16 input_vlan_tag;
  667. + int input_ifindex;
  668. #ifdef CONFIG_NET_SWITCHDEV
  669. /* Set if TX data plane offloading is used towards at least one
  670. --- /dev/null
  671. +++ b/net/bridge/br_private_offload.h
  672. @@ -0,0 +1,23 @@
  673. +#ifndef __BR_OFFLOAD_H
  674. +#define __BR_OFFLOAD_H
  675. +
  676. +bool br_offload_input(struct net_bridge_port *p, struct sk_buff *skb);
  677. +void br_offload_output(struct sk_buff *skb);
  678. +void br_offload_port_state(struct net_bridge_port *p);
  679. +void br_offload_fdb_update(const struct net_bridge_fdb_entry *fdb);
  680. +int br_offload_init(void);
  681. +void br_offload_fini(void);
  682. +int br_offload_set_cache_size(struct net_bridge *br, unsigned long val,
  683. + struct netlink_ext_ack *extack);
  684. +int br_offload_set_cache_reserved(struct net_bridge *br, unsigned long val,
  685. + struct netlink_ext_ack *extack);
  686. +
  687. +static inline void br_offload_skb_disable(struct sk_buff *skb)
  688. +{
  689. + struct br_input_skb_cb *cb = (struct br_input_skb_cb *)skb->cb;
  690. +
  691. + if (cb->offload)
  692. + cb->offload = 0;
  693. +}
  694. +
  695. +#endif
  696. --- a/net/bridge/br_stp.c
  697. +++ b/net/bridge/br_stp.c
  698. @@ -12,6 +12,7 @@
  699. #include "br_private.h"
  700. #include "br_private_stp.h"
  701. +#include "br_private_offload.h"
  702. /* since time values in bpdu are in jiffies and then scaled (1/256)
  703. * before sending, make sure that is at least one STP tick.
  704. @@ -52,6 +53,8 @@ void br_set_state(struct net_bridge_port
  705. (unsigned int) p->port_no, p->dev->name,
  706. br_port_state_names[p->state]);
  707. + br_offload_port_state(p);
  708. +
  709. if (p->br->stp_enabled == BR_KERNEL_STP) {
  710. switch (p->state) {
  711. case BR_STATE_BLOCKING:
  712. --- a/net/bridge/br_sysfs_br.c
  713. +++ b/net/bridge/br_sysfs_br.c
  714. @@ -18,6 +18,7 @@
  715. #include <linux/sched/signal.h>
  716. #include "br_private.h"
  717. +#include "br_private_offload.h"
  718. /* IMPORTANT: new bridge options must be added with netlink support only
  719. * please do not add new sysfs entries
  720. @@ -930,6 +931,38 @@ static ssize_t vlan_stats_per_port_store
  721. static DEVICE_ATTR_RW(vlan_stats_per_port);
  722. #endif
  723. +static ssize_t offload_cache_size_show(struct device *d,
  724. + struct device_attribute *attr,
  725. + char *buf)
  726. +{
  727. + struct net_bridge *br = to_bridge(d);
  728. + return sprintf(buf, "%u\n", br->offload_cache_size);
  729. +}
  730. +
  731. +static ssize_t offload_cache_size_store(struct device *d,
  732. + struct device_attribute *attr,
  733. + const char *buf, size_t len)
  734. +{
  735. + return store_bridge_parm(d, buf, len, br_offload_set_cache_size);
  736. +}
  737. +static DEVICE_ATTR_RW(offload_cache_size);
  738. +
  739. +static ssize_t offload_cache_reserved_show(struct device *d,
  740. + struct device_attribute *attr,
  741. + char *buf)
  742. +{
  743. + struct net_bridge *br = to_bridge(d);
  744. + return sprintf(buf, "%u\n", br->offload_cache_reserved);
  745. +}
  746. +
  747. +static ssize_t offload_cache_reserved_store(struct device *d,
  748. + struct device_attribute *attr,
  749. + const char *buf, size_t len)
  750. +{
  751. + return store_bridge_parm(d, buf, len, br_offload_set_cache_reserved);
  752. +}
  753. +static DEVICE_ATTR_RW(offload_cache_reserved);
  754. +
  755. static struct attribute *bridge_attrs[] = {
  756. &dev_attr_forward_delay.attr,
  757. &dev_attr_hello_time.attr,
  758. @@ -984,6 +1017,8 @@ static struct attribute *bridge_attrs[]
  759. &dev_attr_vlan_stats_enabled.attr,
  760. &dev_attr_vlan_stats_per_port.attr,
  761. #endif
  762. + &dev_attr_offload_cache_size.attr,
  763. + &dev_attr_offload_cache_reserved.attr,
  764. NULL
  765. };
  766. --- a/net/bridge/br_sysfs_if.c
  767. +++ b/net/bridge/br_sysfs_if.c
  768. @@ -241,6 +241,7 @@ BRPORT_ATTR_FLAG(broadcast_flood, BR_BCA
  769. BRPORT_ATTR_FLAG(neigh_suppress, BR_NEIGH_SUPPRESS);
  770. BRPORT_ATTR_FLAG(isolated, BR_ISOLATED);
  771. BRPORT_ATTR_FLAG(bpdu_filter, BR_BPDU_FILTER);
  772. +BRPORT_ATTR_FLAG(offload, BR_OFFLOAD);
  773. #ifdef CONFIG_BRIDGE_IGMP_SNOOPING
  774. static ssize_t show_multicast_router(struct net_bridge_port *p, char *buf)
  775. @@ -295,6 +296,7 @@ static const struct brport_attribute *br
  776. &brport_attr_isolated,
  777. &brport_attr_bpdu_filter,
  778. &brport_attr_backup_port,
  779. + &brport_attr_offload,
  780. NULL
  781. };
  782. --- a/net/bridge/br_vlan_tunnel.c
  783. +++ b/net/bridge/br_vlan_tunnel.c
  784. @@ -15,6 +15,7 @@
  785. #include "br_private.h"
  786. #include "br_private_tunnel.h"
  787. +#include "br_private_offload.h"
  788. static inline int br_vlan_tunid_cmp(struct rhashtable_compare_arg *arg,
  789. const void *ptr)
  790. @@ -180,6 +181,7 @@ void br_handle_ingress_vlan_tunnel(struc
  791. skb_dst_drop(skb);
  792. __vlan_hwaccel_put_tag(skb, p->br->vlan_proto, vlan->vid);
  793. + br_offload_skb_disable(skb);
  794. }
  795. int br_handle_egress_vlan_tunnel(struct sk_buff *skb,
  796. @@ -201,6 +203,7 @@ int br_handle_egress_vlan_tunnel(struct
  797. if (err)
  798. return err;
  799. + br_offload_skb_disable(skb);
  800. tunnel_dst = rcu_dereference(vlan->tinfo.tunnel_dst);
  801. if (tunnel_dst && dst_hold_safe(&tunnel_dst->dst))
  802. skb_dst_set(skb, &tunnel_dst->dst);