1 files changed, 491 insertions, 121 deletions

diff --git a/include/net/ip6_fib.h b/include/net/ip6_fib.h
index 2a601e7da1bf..88b0dd4d8e09 100644
--- a/include/net/ip6_fib.h
+++ b/include/net/ip6_fib.h
@@ -1,13 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
 /*
  *	Linux INET6 implementation 
  *
  *	Authors:
  *	Pedro Roque		<roque@di.fc.ul.pt>	
- *
- *	This program is free software; you can redistribute it and/or
- *      modify it under the terms of the GNU General Public License
- *      as published by the Free Software Foundation; either version
- *      2 of the License, or (at your option) any later version.
  */
 
 #ifndef _IP6_FIB_H
@@ -16,10 +12,15 @@
 #include <linux/ipv6_route.h>
 #include <linux/rtnetlink.h>
 #include <linux/spinlock.h>
+#include <linux/notifier.h>
 #include <net/dst.h>
 #include <net/flow.h>
+#include <net/ip_fib.h>
 #include <net/netlink.h>
 #include <net/inetpeer.h>
+#include <net/fib_notifier.h>
+#include <linux/indirect_call_wrapper.h>
+#include <uapi/linux/bpf.h>
 
 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
 #define FIB6_TABLE_HASHSZ 256
@@ -27,7 +28,10 @@
 #define FIB6_TABLE_HASHSZ 1
 #endif
 
+#define RT6_DEBUG 2
+
 struct rt6_info;
+struct fib6_info;
 
 struct fib6_config {
 	u32		fc_table;
@@ -37,7 +41,11 @@ struct fib6_config {
 	int		fc_ifindex;
 	u32		fc_flags;
 	u32		fc_protocol;
-	u32		fc_type;	/* only 8 bits are used */
+	u16		fc_type;        /* only 8 bits are used */
+	u16		fc_delete_all_nh : 1,
+			fc_ignore_dev_down:1,
+			__unused : 14;
+	u32		fc_nh_id;
 
 	struct in6_addr	fc_dst;
 	struct in6_addr	fc_src;
@@ -51,27 +59,61 @@ struct fib6_config {
 	struct nlattr	*fc_mp;
 
 	struct nl_info	fc_nlinfo;
+	struct nlattr	*fc_encap;
+	u16		fc_encap_type;
+	bool		fc_is_fdb;
 };
 
 struct fib6_node {
-	struct fib6_node	*parent;
-	struct fib6_node	*left;
-	struct fib6_node	*right;
+	struct fib6_node __rcu	*parent;
+	struct fib6_node __rcu	*left;
+	struct fib6_node __rcu	*right;
 #ifdef CONFIG_IPV6_SUBTREES
-	struct fib6_node	*subtree;
+	struct fib6_node __rcu	*subtree;
 #endif
-	struct rt6_info		*leaf;
+	struct fib6_info __rcu	*leaf;
 
 	__u16			fn_bit;		/* bit key */
 	__u16			fn_flags;
-	__u32			fn_sernum;
-	struct rt6_info		*rr_ptr;
+	int			fn_sernum;
+	struct fib6_info __rcu	*rr_ptr;
+	struct rcu_head		rcu;
+};
+
+struct fib6_gc_args {
+	int			timeout;
+	int			more;
 };
 
 #ifndef CONFIG_IPV6_SUBTREES
 #define FIB6_SUBTREE(fn)	NULL
+
+static inline bool fib6_routes_require_src(const struct net *net)
+{
+	return false;
+}
+
+static inline void fib6_routes_require_src_inc(struct net *net) {}
+static inline void fib6_routes_require_src_dec(struct net *net) {}
+
 #else
-#define FIB6_SUBTREE(fn)	((fn)->subtree)
+
+static inline bool fib6_routes_require_src(const struct net *net)
+{
+	return net->ipv6.fib6_routes_require_src > 0;
+}
+
+static inline void fib6_routes_require_src_inc(struct net *net)
+{
+	net->ipv6.fib6_routes_require_src++;
+}
+
+static inline void fib6_routes_require_src_dec(struct net *net)
+{
+	net->ipv6.fib6_routes_require_src--;
+}
+
+#define FIB6_SUBTREE(fn)	(rcu_dereference_protected((fn)->subtree, 1))
 #endif
 
 /*
@@ -86,113 +128,191 @@ struct rt6key {
 
 struct fib6_table;
 
-struct rt6_info {
-	struct dst_entry		dst;
+struct rt6_exception_bucket {
+	struct hlist_head	chain;
+	int			depth;
+};
 
-	/*
-	 * Tail elements of dst_entry (__refcnt etc.)
-	 * and these elements (rarely used in hot path) are in
-	 * the same cache line.
-	 */
-	struct fib6_table		*rt6i_table;
-	struct fib6_node		*rt6i_node;
+struct rt6_exception {
+	struct hlist_node	hlist;
+	struct rt6_info		*rt6i;
+	unsigned long		stamp;
+	struct rcu_head		rcu;
+};
 
-	struct in6_addr			rt6i_gateway;
+#define FIB6_EXCEPTION_BUCKET_SIZE_SHIFT 10
+#define FIB6_EXCEPTION_BUCKET_SIZE (1 << FIB6_EXCEPTION_BUCKET_SIZE_SHIFT)
+#define FIB6_MAX_DEPTH 5
+
+struct fib6_nh {
+	struct fib_nh_common	nh_common;
+
+#ifdef CONFIG_IPV6_ROUTER_PREF
+	unsigned long		last_probe;
+#endif
+
+	struct rt6_info * __percpu *rt6i_pcpu;
+	struct rt6_exception_bucket __rcu *rt6i_exception_bucket;
+};
+
+struct fib6_info {
+	struct fib6_table		*fib6_table;
+	struct fib6_info __rcu		*fib6_next;
+	struct fib6_node __rcu		*fib6_node;
 
 	/* Multipath routes:
-	 * siblings is a list of rt6_info that have the the same metric/weight,
+	 * siblings is a list of fib6_info that have the same metric/weight,
 	 * destination, but not the same gateway. nsiblings is just a cache
 	 * to speed up lookup.
 	 */
-	struct list_head		rt6i_siblings;
-	unsigned int			rt6i_nsiblings;
+	union {
+		struct list_head	fib6_siblings;
+		struct list_head	nh_list;
+	};
+	unsigned int			fib6_nsiblings;
+
+	refcount_t			fib6_ref;
+	unsigned long			expires;
+
+	struct hlist_node		gc_link;
+
+	struct dst_metrics		*fib6_metrics;
+#define fib6_pmtu		fib6_metrics->metrics[RTAX_MTU-1]
+
+	struct rt6key			fib6_dst;
+	u32				fib6_flags;
+	struct rt6key			fib6_src;
+	struct rt6key			fib6_prefsrc;
+
+	u32				fib6_metric;
+	u8				fib6_protocol;
+	u8				fib6_type;
+
+	u8				offload;
+	u8				trap;
+	u8				offload_failed;
+
+	u8				should_flush:1,
+					dst_nocount:1,
+					dst_nopolicy:1,
+					fib6_destroying:1,
+					unused:4;
+
+	struct list_head		purge_link;
+	struct rcu_head			rcu;
+	struct nexthop			*nh;
+	struct fib6_nh			fib6_nh[];
+};
 
-	atomic_t			rt6i_ref;
+struct rt6_info {
+	struct dst_entry		dst;
+	struct fib6_info __rcu		*from;
+	int				sernum;
 
-	/* These are in a separate cache line. */
-	struct rt6key			rt6i_dst ____cacheline_aligned_in_smp;
-	u32				rt6i_flags;
+	struct rt6key			rt6i_dst;
 	struct rt6key			rt6i_src;
-	struct rt6key			rt6i_prefsrc;
-	u32				rt6i_metric;
-
+	struct in6_addr			rt6i_gateway;
 	struct inet6_dev		*rt6i_idev;
-	unsigned long			_rt6i_peer;
-
-	u32				rt6i_genid;
+	u32				rt6i_flags;
 
 	/* more non-fragment space at head required */
 	unsigned short			rt6i_nfheader_len;
+};
 
-	u8				rt6i_protocol;
+struct fib6_result {
+	struct fib6_nh		*nh;
+	struct fib6_info	*f6i;
+	u32			fib6_flags;
+	u8			fib6_type;
+	struct rt6_info		*rt6;
 };
 
-static inline struct inet_peer *rt6_peer_ptr(struct rt6_info *rt)
-{
-	return inetpeer_ptr(rt->_rt6i_peer);
-}
+#define for_each_fib6_node_rt_rcu(fn)					\
+	for (rt = rcu_dereference((fn)->leaf); rt;			\
+	     rt = rcu_dereference(rt->fib6_next))
 
-static inline bool rt6_has_peer(struct rt6_info *rt)
-{
-	return inetpeer_ptr_is_peer(rt->_rt6i_peer);
-}
+#define for_each_fib6_walker_rt(w)					\
+	for (rt = (w)->leaf; rt;					\
+	     rt = rcu_dereference_protected(rt->fib6_next, 1))
 
-static inline void __rt6_set_peer(struct rt6_info *rt, struct inet_peer *peer)
-{
-	__inetpeer_ptr_set_peer(&rt->_rt6i_peer, peer);
-}
+#define dst_rt6_info(_ptr) container_of_const(_ptr, struct rt6_info, dst)
 
-static inline bool rt6_set_peer(struct rt6_info *rt, struct inet_peer *peer)
+static inline struct inet6_dev *ip6_dst_idev(const struct dst_entry *dst)
 {
-	return inetpeer_ptr_set_peer(&rt->_rt6i_peer, peer);
+	return dst_rt6_info(dst)->rt6i_idev;
 }
 
-static inline void rt6_init_peer(struct rt6_info *rt, struct inet_peer_base *base)
+static inline bool fib6_requires_src(const struct fib6_info *rt)
 {
-	inetpeer_init_ptr(&rt->_rt6i_peer, base);
+	return rt->fib6_src.plen > 0;
 }
 
-static inline void rt6_transfer_peer(struct rt6_info *rt, struct rt6_info *ort)
+/* The callers should hold f6i->fib6_table->tb6_lock if a route has ever
+ * been added to a table before.
+ */
+static inline void fib6_clean_expires(struct fib6_info *f6i)
 {
-	inetpeer_transfer_peer(&rt->_rt6i_peer, &ort->_rt6i_peer);
+	f6i->fib6_flags &= ~RTF_EXPIRES;
+	f6i->expires = 0;
 }
 
-static inline struct inet6_dev *ip6_dst_idev(struct dst_entry *dst)
+/* The callers should hold f6i->fib6_table->tb6_lock if a route has ever
+ * been added to a table before.
+ */
+static inline void fib6_set_expires(struct fib6_info *f6i,
+				    unsigned long expires)
 {
-	return ((struct rt6_info *)dst)->rt6i_idev;
+	f6i->expires = expires;
+	f6i->fib6_flags |= RTF_EXPIRES;
 }
 
-static inline void rt6_clean_expires(struct rt6_info *rt)
+static inline bool fib6_check_expired(const struct fib6_info *f6i)
 {
-	rt->rt6i_flags &= ~RTF_EXPIRES;
+	if (f6i->fib6_flags & RTF_EXPIRES)
+		return time_after(jiffies, f6i->expires);
+	return false;
 }
 
-static inline void rt6_set_expires(struct rt6_info *rt, unsigned long expires)
+/* Function to safely get fn->fn_sernum for passed in rt
+ * and store result in passed in cookie.
+ * Return true if we can get cookie safely
+ * Return false if not
+ */
+static inline bool fib6_get_cookie_safe(const struct fib6_info *f6i,
+					u32 *cookie)
 {
-	rt->dst.expires = expires;
-	rt->rt6i_flags |= RTF_EXPIRES;
-}
+	struct fib6_node *fn;
+	bool status = false;
 
-static inline void rt6_update_expires(struct rt6_info *rt0, int timeout)
-{
-	struct rt6_info *rt;
+	fn = rcu_dereference(f6i->fib6_node);
 
-	for (rt = rt0; rt && !(rt->rt6i_flags & RTF_EXPIRES);
-	     rt = (struct rt6_info *)rt->dst.from);
-	if (rt && rt != rt0)
-		rt0->dst.expires = rt->dst.expires;
+	if (fn) {
+		*cookie = READ_ONCE(fn->fn_sernum);
+		/* pairs with smp_wmb() in __fib6_update_sernum_upto_root() */
+		smp_rmb();
+		status = true;
+	}
 
-	dst_set_expires(&rt0->dst, timeout);
-	rt0->rt6i_flags |= RTF_EXPIRES;
+	return status;
 }
 
-static inline void rt6_set_from(struct rt6_info *rt, struct rt6_info *from)
+static inline u32 rt6_get_cookie(const struct rt6_info *rt)
 {
-	struct dst_entry *new = (struct dst_entry *) from;
+	struct fib6_info *from;
+	u32 cookie = 0;
+
+	if (rt->sernum)
+		return rt->sernum;
 
-	rt->rt6i_flags &= ~RTF_EXPIRES;
-	dst_hold(new);
-	rt->dst.from = new;
+	rcu_read_lock();
+
+	from = rcu_dereference(rt->from);
+	if (from)
+		fib6_get_cookie_safe(from, &cookie);
+
+	rcu_read_unlock();
+
+	return cookie;
 }
 
 static inline void ip6_rt_put(struct rt6_info *rt)
@@ -204,25 +324,58 @@ static inline void ip6_rt_put(struct rt6_info *rt)
 	dst_release(&rt->dst);
 }
 
-struct fib6_walker_t {
+struct fib6_info *fib6_info_alloc(gfp_t gfp_flags, bool with_fib6_nh);
+void fib6_info_destroy_rcu(struct rcu_head *head);
+
+static inline void fib6_info_hold(struct fib6_info *f6i)
+{
+	refcount_inc(&f6i->fib6_ref);
+}
+
+static inline bool fib6_info_hold_safe(struct fib6_info *f6i)
+{
+	return refcount_inc_not_zero(&f6i->fib6_ref);
+}
+
+static inline void fib6_info_release(struct fib6_info *f6i)
+{
+	if (f6i && refcount_dec_and_test(&f6i->fib6_ref)) {
+		DEBUG_NET_WARN_ON_ONCE(!hlist_unhashed(&f6i->gc_link));
+		call_rcu_hurry(&f6i->rcu, fib6_info_destroy_rcu);
+	}
+}
+
+enum fib6_walk_state {
+#ifdef CONFIG_IPV6_SUBTREES
+	FWS_S,
+#endif
+	FWS_L,
+	FWS_R,
+	FWS_C,
+	FWS_U
+};
+
+struct fib6_walker {
 	struct list_head lh;
 	struct fib6_node *root, *node;
-	struct rt6_info *leaf;
-	unsigned char state;
-	unsigned char prune;
+	struct fib6_info *leaf;
+	enum fib6_walk_state state;
 	unsigned int skip;
 	unsigned int count;
-	int (*func)(struct fib6_walker_t *);
+	unsigned int skip_in_node;
+	int (*func)(struct fib6_walker *);
 	void *args;
 };
 
 struct rt6_statistics {
-	__u32		fib_nodes;
-	__u32		fib_route_nodes;
-	__u32		fib_rt_alloc;		/* permanent routes	*/
-	__u32		fib_rt_entries;		/* rt entries in table	*/
-	__u32		fib_rt_cache;		/* cache routes		*/
-	__u32		fib_discarded_routes;
+	__u32		fib_nodes;		/* all fib6 nodes */
+	__u32		fib_route_nodes;	/* intermediate nodes */
+	__u32		fib_rt_entries;		/* rt entries in fib table */
+	__u32		fib_rt_cache;		/* cached rt entries in exception table */
+	__u32		fib_discarded_routes;	/* total number of routes delete */
+
+	/* The following stat is not protected by any lock */
+	atomic_t	fib_rt_alloc;		/* total number of routes alloced */
 };
 
 #define RTN_TL_ROOT	0x0001
@@ -238,9 +391,13 @@ struct rt6_statistics {
 struct fib6_table {
 	struct hlist_node	tb6_hlist;
 	u32			tb6_id;
-	rwlock_t		tb6_lock;
+	spinlock_t		tb6_lock;
 	struct fib6_node	tb6_root;
 	struct inet_peer_base	tb6_peers;
+	unsigned int		flags;
+	unsigned int		fib_seq; /* writes protected by rtnl_mutex */
+	struct hlist_head       tb6_gc_hlist;	/* GC candidates */
+#define RT6_TABLE_HAS_DFLT_ROUTER	BIT(0)
 };
 
 #define RT6_TABLE_UNSPEC	RT_TABLE_UNSPEC
@@ -261,55 +418,248 @@ struct fib6_table {
 
 typedef struct rt6_info *(*pol_lookup_t)(struct net *,
 					 struct fib6_table *,
-					 struct flowi6 *, int);
+					 struct flowi6 *,
+					 const struct sk_buff *, int);
+
+struct fib6_entry_notifier_info {
+	struct fib_notifier_info info; /* must be first */
+	struct fib6_info *rt;
+	unsigned int nsiblings;
+};
 
 /*
  *	exported functions
  */
 
-extern struct fib6_table        *fib6_get_table(struct net *net, u32 id);
-extern struct fib6_table        *fib6_new_table(struct net *net, u32 id);
-extern struct dst_entry         *fib6_rule_lookup(struct net *net,
-						  struct flowi6 *fl6, int flags,
-						  pol_lookup_t lookup);
+struct fib6_table *fib6_get_table(struct net *net, u32 id);
+struct fib6_table *fib6_new_table(struct net *net, u32 id);
+struct dst_entry *fib6_rule_lookup(struct net *net, struct flowi6 *fl6,
+				   const struct sk_buff *skb,
+				   int flags, pol_lookup_t lookup);
+
+/* called with rcu lock held; can return error pointer
+ * caller needs to select path
+ */
+int fib6_lookup(struct net *net, int oif, struct flowi6 *fl6,
+		struct fib6_result *res, int flags);
+
+/* called with rcu lock held; caller needs to select path */
+int fib6_table_lookup(struct net *net, struct fib6_table *table,
+		      int oif, struct flowi6 *fl6, struct fib6_result *res,
+		      int strict);
+
+void fib6_select_path(const struct net *net, struct fib6_result *res,
+		      struct flowi6 *fl6, int oif, bool have_oif_match,
+		      const struct sk_buff *skb, int strict);
+struct fib6_node *fib6_node_lookup(struct fib6_node *root,
+				   const struct in6_addr *daddr,
+				   const struct in6_addr *saddr);
+
+struct fib6_node *fib6_locate(struct fib6_node *root,
+			      const struct in6_addr *daddr, int dst_len,
+			      const struct in6_addr *saddr, int src_len,
+			      bool exact_match);
+
+void fib6_clean_all(struct net *net, int (*func)(struct fib6_info *, void *arg),
+		    void *arg);
+void fib6_clean_all_skip_notify(struct net *net,
+				int (*func)(struct fib6_info *, void *arg),
+				void *arg);
+
+int fib6_add(struct fib6_node *root, struct fib6_info *rt,
+	     struct nl_info *info, struct netlink_ext_ack *extack);
+int fib6_del(struct fib6_info *rt, struct nl_info *info);
+
+static inline
+void rt6_get_prefsrc(const struct rt6_info *rt, struct in6_addr *addr)
+{
+	const struct fib6_info *from;
 
-extern struct fib6_node		*fib6_lookup(struct fib6_node *root,
-					     const struct in6_addr *daddr,
-					     const struct in6_addr *saddr);
+	rcu_read_lock();
+
+	from = rcu_dereference(rt->from);
+	if (from)
+		*addr = from->fib6_prefsrc.addr;
+	else
+		*addr = in6addr_any;
+
+	rcu_read_unlock();
+}
+
+int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh,
+		 struct fib6_config *cfg, gfp_t gfp_flags,
+		 struct netlink_ext_ack *extack);
+void fib6_nh_release(struct fib6_nh *fib6_nh);
+void fib6_nh_release_dsts(struct fib6_nh *fib6_nh);
+
+int call_fib6_entry_notifiers(struct net *net,
+			      enum fib_event_type event_type,
+			      struct fib6_info *rt,
+			      struct netlink_ext_ack *extack);
+int call_fib6_multipath_entry_notifiers(struct net *net,
+					enum fib_event_type event_type,
+					struct fib6_info *rt,
+					unsigned int nsiblings,
+					struct netlink_ext_ack *extack);
+int call_fib6_entry_notifiers_replace(struct net *net, struct fib6_info *rt);
+void fib6_rt_update(struct net *net, struct fib6_info *rt,
+		    struct nl_info *info);
+void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info,
+		     unsigned int flags);
+
+void fib6_run_gc(unsigned long expires, struct net *net, bool force);
+
+void fib6_gc_cleanup(void);
+
+int fib6_init(void);
+
+/* Add the route to the gc list if it is not already there
+ *
+ * The callers should hold f6i->fib6_table->tb6_lock.
+ */
+static inline void fib6_add_gc_list(struct fib6_info *f6i)
+{
+	/* If fib6_node is null, the f6i is not in (or removed from) the
+	 * table.
+	 *
+	 * There is a gap between finding the f6i from the table and
+	 * calling this function without the protection of the tb6_lock.
+	 * This check makes sure the f6i is not added to the gc list when
+	 * it is not on the table.
+	 */
+	if (!rcu_dereference_protected(f6i->fib6_node,
+				       lockdep_is_held(&f6i->fib6_table->tb6_lock)))
+		return;
+
+	if (hlist_unhashed(&f6i->gc_link))
+		hlist_add_head(&f6i->gc_link, &f6i->fib6_table->tb6_gc_hlist);
+}
+
+/* Remove the route from the gc list if it is on the list.
+ *
+ * The callers should hold f6i->fib6_table->tb6_lock.
+ */
+static inline void fib6_remove_gc_list(struct fib6_info *f6i)
+{
+	if (!hlist_unhashed(&f6i->gc_link))
+		hlist_del_init(&f6i->gc_link);
+}
 
-struct fib6_node		*fib6_locate(struct fib6_node *root,
-					     const struct in6_addr *daddr, int dst_len,
-					     const struct in6_addr *saddr, int src_len);
+struct ipv6_route_iter {
+	struct seq_net_private p;
+	struct fib6_walker w;
+	loff_t skip;
+	struct fib6_table *tbl;
+	int sernum;
+};
 
-extern void			fib6_clean_all_ro(struct net *net,
-					       int (*func)(struct rt6_info *, void *arg),
-					       int prune, void *arg);
+extern const struct seq_operations ipv6_route_seq_ops;
 
-extern void			fib6_clean_all(struct net *net,
-					       int (*func)(struct rt6_info *, void *arg),
-					       int prune, void *arg);
+int call_fib6_notifier(struct notifier_block *nb,
+		       enum fib_event_type event_type,
+		       struct fib_notifier_info *info);
+int call_fib6_notifiers(struct net *net, enum fib_event_type event_type,
+			struct fib_notifier_info *info);
 
-extern int			fib6_add(struct fib6_node *root,
-					 struct rt6_info *rt,
-					 struct nl_info *info);
+int __net_init fib6_notifier_init(struct net *net);
+void __net_exit fib6_notifier_exit(struct net *net);
 
-extern int			fib6_del(struct rt6_info *rt,
-					 struct nl_info *info);
+unsigned int fib6_tables_seq_read(const struct net *net);
+int fib6_tables_dump(struct net *net, struct notifier_block *nb,
+		     struct netlink_ext_ack *extack);
 
-extern void			inet6_rt_notify(int event, struct rt6_info *rt,
-						struct nl_info *info);
+void fib6_update_sernum(struct net *net, struct fib6_info *rt);
+void fib6_update_sernum_upto_root(struct net *net, struct fib6_info *rt);
+void fib6_update_sernum_stub(struct net *net, struct fib6_info *f6i);
 
-extern void			fib6_run_gc(unsigned long expires,
-					    struct net *net);
+void fib6_metric_set(struct fib6_info *f6i, int metric, u32 val);
+static inline bool fib6_metric_locked(struct fib6_info *f6i, int metric)
+{
+	return !!(f6i->fib6_metrics->metrics[RTAX_LOCK - 1] & (1 << metric));
+}
+void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i,
+			    bool offload, bool trap, bool offload_failed);
 
-extern void			fib6_gc_cleanup(void);
+#if IS_BUILTIN(CONFIG_IPV6) && defined(CONFIG_BPF_SYSCALL)
+struct bpf_iter__ipv6_route {
+	__bpf_md_ptr(struct bpf_iter_meta *, meta);
+	__bpf_md_ptr(struct fib6_info *, rt);
+};
+#endif
 
-extern int			fib6_init(void);
+INDIRECT_CALLABLE_DECLARE(struct rt6_info *ip6_pol_route_output(struct net *net,
+					     struct fib6_table *table,
+					     struct flowi6 *fl6,
+					     const struct sk_buff *skb,
+					     int flags));
+INDIRECT_CALLABLE_DECLARE(struct rt6_info *ip6_pol_route_input(struct net *net,
+					     struct fib6_table *table,
+					     struct flowi6 *fl6,
+					     const struct sk_buff *skb,
+					     int flags));
+INDIRECT_CALLABLE_DECLARE(struct rt6_info *__ip6_route_redirect(struct net *net,
+					     struct fib6_table *table,
+					     struct flowi6 *fl6,
+					     const struct sk_buff *skb,
+					     int flags));
+INDIRECT_CALLABLE_DECLARE(struct rt6_info *ip6_pol_route_lookup(struct net *net,
+					     struct fib6_table *table,
+					     struct flowi6 *fl6,
+					     const struct sk_buff *skb,
+					     int flags));
+static inline struct rt6_info *pol_lookup_func(pol_lookup_t lookup,
+						struct net *net,
+						struct fib6_table *table,
+						struct flowi6 *fl6,
+						const struct sk_buff *skb,
+						int flags)
+{
+	return INDIRECT_CALL_4(lookup,
+			       ip6_pol_route_output,
+			       ip6_pol_route_input,
+			       ip6_pol_route_lookup,
+			       __ip6_route_redirect,
+			       net, table, fl6, skb, flags);
+}
 
 #ifdef CONFIG_IPV6_MULTIPLE_TABLES
-extern int			fib6_rules_init(void);
-extern void			fib6_rules_cleanup(void);
+static inline bool fib6_has_custom_rules(const struct net *net)
+{
+	return net->ipv6.fib6_has_custom_rules;
+}
+
+int fib6_rules_init(void);
+void fib6_rules_cleanup(void);
+bool fib6_rule_default(const struct fib_rule *rule);
+int fib6_rules_dump(struct net *net, struct notifier_block *nb,
+		    struct netlink_ext_ack *extack);
+unsigned int fib6_rules_seq_read(const struct net *net);
+
+static inline bool fib6_rules_early_flow_dissect(struct net *net,
+						 struct sk_buff *skb,
+						 struct flowi6 *fl6,
+						 struct flow_keys *flkeys)
+{
+	unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
+
+	if (!net->ipv6.fib6_rules_require_fldissect)
+		return false;
+
+	memset(flkeys, 0, sizeof(*flkeys));
+	__skb_flow_dissect(net, skb, &flow_keys_dissector,
+			   flkeys, NULL, 0, 0, 0, flag);
+
+	fl6->fl6_sport = flkeys->ports.src;
+	fl6->fl6_dport = flkeys->ports.dst;
+	fl6->flowi6_proto = flkeys->basic.ip_proto;
+
+	return true;
+}
 #else
+static inline bool fib6_has_custom_rules(const struct net *net)
+{
+	return false;
+}
 static inline int               fib6_rules_init(void)
 {
 	return 0;
@@ -318,5 +668,25 @@ static inline void              fib6_rules_cleanup(void)
 {
 	return ;
 }
+static inline bool fib6_rule_default(const struct fib_rule *rule)
+{
+	return true;
+}
+static inline int fib6_rules_dump(struct net *net, struct notifier_block *nb,
+				  struct netlink_ext_ack *extack)
+{
+	return 0;
+}
+static inline unsigned int fib6_rules_seq_read(const struct net *net)
+{
+	return 0;
+}
+static inline bool fib6_rules_early_flow_dissect(struct net *net,
+						 struct sk_buff *skb,
+						 struct flowi6 *fl6,
+						 struct flow_keys *flkeys)
+{
+	return false;
+}
 #endif
 #endif